From 551dcbd86fa1e7e57398ef04eba3694d2ce49035 Mon Sep 17 00:00:00 2001 From: Edoardo Pirovano Date: Thu, 24 Feb 2022 17:12:08 +0000 Subject: [PATCH 1/3] Expose per-query structured evaluator logs --- extensions/ql-vscode/CHANGELOG.md | 1 + extensions/ql-vscode/package.json | 26 ++++++++ extensions/ql-vscode/src/cli.ts | 26 ++++++++ extensions/ql-vscode/src/extension.ts | 2 + extensions/ql-vscode/src/pure/messages.ts | 59 +++++++++++++++++++ extensions/ql-vscode/src/query-history.ts | 54 +++++++++++++++++ extensions/ql-vscode/src/query-results.ts | 1 + .../ql-vscode/src/queryserver-client.ts | 6 +- extensions/ql-vscode/src/run-queries.ts | 23 +++++++- 9 files changed, 195 insertions(+), 3 deletions(-) diff --git a/extensions/ql-vscode/CHANGELOG.md b/extensions/ql-vscode/CHANGELOG.md index ac8b15df023..be205c6fe99 100644 --- a/extensions/ql-vscode/CHANGELOG.md +++ b/extensions/ql-vscode/CHANGELOG.md @@ -15,6 +15,7 @@ No user facing changes. - Fix a bug where queries took a long time to run if there are no folders in the workspace. [#1157](https://github.com/github/vscode-codeql/pull/1157) - [BREAKING CHANGE] The `codeQL.runningQueries.customLogDirectory` setting is deprecated and no longer has any function. Instead, all query log files will be stored in the query history directory, next to the query results. [#1178](https://github.com/github/vscode-codeql/pull/1178) - Add a _Open query directory_ command for query items. This command opens the directory containing all artifacts for a query. [#1179](https://github.com/github/vscode-codeql/pull/1179) +- Add options to display evaluator logs for a given query run. Some information that was previously found in the query server output may now be found here. [#1186](https://github.com/github/vscode-codeql/pull/1186) ## 1.5.11 - 10 February 2022 diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 379932b3256..06f592626bf 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -522,6 +522,14 @@ "command": "codeQLQueryHistory.openQueryDirectory", "title": "Open query directory" }, + { + "command": "codeQLQueryHistory.showEvalLog", + "title": "Show Evaluator Log (Raw)" + }, + { + "command": "codeQLQueryHistory.showEvalLogSummary", + "title": "Show Evaluator Log (Summary)" + }, { "command": "codeQLQueryHistory.cancel", "title": "Cancel" @@ -725,6 +733,16 @@ "group": "9_qlCommands", "when": "view == codeQLQueryHistory && !hasRemoteServer" }, + { + "command": "codeQLQueryHistory.showEvalLog", + "group": "9_qlCommands", + "when": "viewItem == rawResultsItem || viewItem == interpretedResultsItem || viewItem == cancelledResultsItem" + }, + { + "command": "codeQLQueryHistory.showEvalLogSummary", + "group": "9_qlCommands", + "when": "viewItem == rawResultsItem || viewItem == interpretedResultsItem || viewItem == cancelledResultsItem" + }, { "command": "codeQLQueryHistory.showQueryText", "group": "9_qlCommands", @@ -924,6 +942,14 @@ "command": "codeQLQueryHistory.showQueryLog", "when": "false" }, + { + "command": "codeQLQueryHistory.showEvalLog", + "when": "false" + }, + { + "command": "codeQLQueryHistory.showEvalLogSummary", + "when": "false" + }, { "command": "codeQLQueryHistory.openQueryDirectory", "when": "false" diff --git a/extensions/ql-vscode/src/cli.ts b/extensions/ql-vscode/src/cli.ts index 7ef6441720b..f8e7d654008 100644 --- a/extensions/ql-vscode/src/cli.ts +++ b/extensions/ql-vscode/src/cli.ts @@ -665,6 +665,23 @@ export class CodeQLCliServer implements Disposable { return await this.runCodeQlCliCommand(['generate', 'query-help'], subcommandArgs, `Generating qhelp in markdown format at ${outputDirectory}`); } + /** + * Generate a summary of an evaluation log. + * @param inputPath The path of an evaluation event log. + * @param outputPath The path to write a human-readable summary of it to. + */ + async generateLogSummary( + inputPath: string, + outputPath: string, + ): Promise { + const subcommandArgs = [ + '--format=text', + inputPath, + outputPath + ]; + return await this.runCodeQlCliCommand(['generate', 'log-summary'], subcommandArgs, 'Generating log summary'); + } + /** * Gets the results from a bqrs. * @param bqrsPath The path to the bqrs. @@ -1256,6 +1273,11 @@ export class CliVersionConstraint { */ public static CLI_VERSION_WITH_STRUCTURED_EVAL_LOG = new SemVer('2.8.2'); + /** + * CLI version that supports rotating structured logs to produce one per query. + */ + public static CLI_VERSION_WITH_PER_QUERY_EVAL_LOG = new SemVer('2.8.4'); + constructor(private readonly cli: CodeQLCliServer) { /**/ } @@ -1315,4 +1337,8 @@ export class CliVersionConstraint { async supportsStructuredEvalLog() { return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_STRUCTURED_EVAL_LOG); } + + async supportsPerQueryEvalLog() { + return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG); + } } diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts index c3eb3f83221..b4c0c671438 100644 --- a/extensions/ql-vscode/src/extension.ts +++ b/extensions/ql-vscode/src/extension.ts @@ -535,6 +535,8 @@ async function activateWithInstalledDistribution( queryStorageDir, progress, source.token, + undefined, + item, ); item.completeThisQuery(completedQueryInfo); await showResultsForCompletedQuery(item as CompletedLocalQueryInfo, WebviewReveal.NotForced); diff --git a/extensions/ql-vscode/src/pure/messages.ts b/extensions/ql-vscode/src/pure/messages.ts index 0341b690cf2..d6f95a6f16a 100644 --- a/extensions/ql-vscode/src/pure/messages.ts +++ b/extensions/ql-vscode/src/pure/messages.ts @@ -646,6 +646,35 @@ export interface ClearCacheParams { */ dryRun: boolean; } + +/** + * Parameters to start a new structured log + */ + export interface StartLogParams { + /** + * The dataset for which we want to start a new structured log + */ + db: Dataset; + /** + * The path where we want to place the new structured log + */ + logPath: string; +} + +/** + * Parameters to terminate a structured log + */ + export interface EndLogParams { + /** + * The dataset for which we want to terminated the log + */ + db: Dataset; + /** + * The path of the log to terminate, will be a no-op if we aren't logging here + */ + logPath: string; +} + /** * Parameters for trimming the cache of a dataset */ @@ -682,6 +711,26 @@ export interface ClearCacheResult { deletionMessage: string; } +/** + * The result of starting a new structured log. + */ +export interface StartLogResult { + /** + * A user friendly message saying what happened. + */ + outcomeMessage: string; +} + +/** + * The result of terminating a structured. + */ +export interface EndLogResult { + /** + * A user friendly message saying what happened. + */ + outcomeMessage: string; +} + /** * Parameters for running a set of queries */ @@ -1018,6 +1067,16 @@ export const compileUpgrade = new rpc.RequestType, CompileUpgradeSequenceResult, void, void>('compilation/compileUpgradeSequence'); +/** + * Start a new structured log in the evaluator, terminating the previous one if it exists + */ + export const startLog = new rpc.RequestType, StartLogResult, void, void>('evaluation/startLog'); + +/** + * Terminate a structured log in the evaluator. Is a no-op if we aren't logging to the given location + */ + export const endLog = new rpc.RequestType, EndLogResult, void, void>('evaluation/endLog'); + /** * Clear the cache of a dataset */ diff --git a/extensions/ql-vscode/src/query-history.ts b/extensions/ql-vscode/src/query-history.ts index 4f3438e0f28..22a14fa9ac4 100644 --- a/extensions/ql-vscode/src/query-history.ts +++ b/extensions/ql-vscode/src/query-history.ts @@ -34,6 +34,8 @@ import { DatabaseManager } from './databases'; import { registerQueryHistoryScubber } from './query-history-scrubber'; import { QueryStatus } from './query-status'; import { slurpQueryHistory, splatQueryHistory } from './query-serialization'; +import * as fs from 'fs-extra'; +import { CliVersionConstraint } from './cli'; /** * query-history.ts @@ -406,6 +408,18 @@ export class QueryHistoryManager extends DisposableObject { this.handleOpenQueryDirectory.bind(this) ) ); + this.push( + commandRunner( + 'codeQLQueryHistory.showEvalLog', + this.handleShowEvalLog.bind(this) + ) + ); + this.push( + commandRunner( + 'codeQLQueryHistory.showEvalLogSummary', + this.handleShowEvalLogSummary.bind(this) + ) + ); this.push( commandRunner( 'codeQLQueryHistory.cancel', @@ -744,6 +758,46 @@ export class QueryHistoryManager extends DisposableObject { } } } + + private warnNoEvalLog() { + void showAndLogWarningMessage('No evaluator log is available for this run. Perhaps it failed before evaluation, or you are running with a version of CodeQL before ' + CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG + '?'); + } + + async handleShowEvalLog( + singleItem: QueryHistoryInfo, + multiSelect: QueryHistoryInfo[] + ) { + // Local queries only + if (!this.assertSingleQuery(multiSelect) || singleItem?.t !== 'local') { + return; + } + + if (singleItem.evalLogLocation) { + await this.tryOpenExternalFile(singleItem.evalLogLocation); + } else { + this.warnNoEvalLog(); + } + } + + async handleShowEvalLogSummary( + singleItem: QueryHistoryInfo, + multiSelect: QueryHistoryInfo[] + ) { + // Local queries only + if (!this.assertSingleQuery(multiSelect) || singleItem?.t !== 'local') { + return; + } + + if (singleItem.evalLogLocation) { + const summaryLocation = singleItem.evalLogLocation + '.summary'; + if (!fs.existsSync(summaryLocation)) { + await this.qs.cliServer.generateLogSummary(singleItem.evalLogLocation, summaryLocation); + } + await this.tryOpenExternalFile(summaryLocation); + } else { + this.warnNoEvalLog(); + } + } async handleCancel( singleItem: QueryHistoryInfo, diff --git a/extensions/ql-vscode/src/query-results.ts b/extensions/ql-vscode/src/query-results.ts index 53313332f8f..7a2426a13f6 100644 --- a/extensions/ql-vscode/src/query-results.ts +++ b/extensions/ql-vscode/src/query-results.ts @@ -216,6 +216,7 @@ export class LocalQueryInfo { public failureReason: string | undefined; public completedQuery: CompletedQueryInfo | undefined; + public evalLogLocation: string | undefined; private config: QueryHistoryConfig | undefined; /** diff --git a/extensions/ql-vscode/src/queryserver-client.ts b/extensions/ql-vscode/src/queryserver-client.ts index 266d3e66c65..23549068d12 100644 --- a/extensions/ql-vscode/src/queryserver-client.ts +++ b/extensions/ql-vscode/src/queryserver-client.ts @@ -146,7 +146,7 @@ export class QueryServerClient extends DisposableObject { args.push('--require-db-registration'); } - if (await this.cliServer.cliConstraints.supportsOldEvalStats()) { + if (await this.cliServer.cliConstraints.supportsOldEvalStats() && !(await this.cliServer.cliConstraints.supportsPerQueryEvalLog())) { args.push('--old-eval-stats'); } @@ -258,3 +258,7 @@ export class QueryServerClient extends DisposableObject { export function findQueryLogFile(resultPath: string): string { return path.join(resultPath, 'query.log'); } + +export function findQueryStructLogFile(resultPath: string): string { + return path.join(resultPath, 'evaluator-log.jsonl'); +} diff --git a/extensions/ql-vscode/src/run-queries.ts b/extensions/ql-vscode/src/run-queries.ts index 9acd7fda1cb..6c47abb9f61 100644 --- a/extensions/ql-vscode/src/run-queries.ts +++ b/extensions/ql-vscode/src/run-queries.ts @@ -29,7 +29,7 @@ import { ProgressCallback, UserCancellationException } from './commandRunner'; import { DatabaseInfo, QueryMetadata } from './pure/interface-types'; import { logger } from './logging'; import * as messages from './pure/messages'; -import { InitialQueryInfo } from './query-results'; +import { InitialQueryInfo, LocalQueryInfo } from './query-results'; import * as qsClient from './queryserver-client'; import { isQuickQueryPath } from './quick-query'; import { compileDatabaseUpgradeSequence, hasNondestructiveUpgradeCapabilities, upgradeDatabaseExplicit } from './upgrades'; @@ -95,6 +95,10 @@ export class QueryEvaluationInfo { return qsClient.findQueryLogFile(this.querySaveDir); } + get structLogPath() { + return qsClient.findQueryStructLogFile(this.querySaveDir); + } + get resultsPaths() { return { resultsPath: path.join(this.querySaveDir, 'results.bqrs'), @@ -125,6 +129,7 @@ export class QueryEvaluationInfo { dbItem: DatabaseItem, progress: ProgressCallback, token: CancellationToken, + queryInfo?: LocalQueryInfo, ): Promise { if (!dbItem.contents || dbItem.error) { throw new Error('Can\'t run query on invalid database.'); @@ -156,6 +161,12 @@ export class QueryEvaluationInfo { dbDir: dbItem.contents.datasetUri.fsPath, workingSet: 'default' }; + if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) { + await qs.sendRequest(messages.startLog, { + db: dataset, + logPath: this.structLogPath, + }); + } const params: messages.EvaluateQueriesParams = { db: dataset, evaluateId: callbackId, @@ -172,6 +183,13 @@ export class QueryEvaluationInfo { } } finally { qs.unRegisterCallback(callbackId); + if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) { + await qs.sendRequest(messages.endLog, { + db: dataset, + logPath: this.structLogPath, + }); + queryInfo.evalLogLocation = this.structLogPath; + } } return result || { evaluationTime: 0, @@ -658,6 +676,7 @@ export async function compileAndRunQueryAgainstDatabase( progress: ProgressCallback, token: CancellationToken, templates?: messages.TemplateDefinitions, + queryInfo?: LocalQueryInfo, ): Promise { if (!dbItem.contents || !dbItem.contents.dbSchemeUri) { throw new Error(`Database ${dbItem.databaseUri} does not have a CodeQL database scheme.`); @@ -743,7 +762,7 @@ export async function compileAndRunQueryAgainstDatabase( } if (errors.length === 0) { - const result = await query.run(qs, upgradeQlo, availableMlModels, dbItem, progress, token); + const result = await query.run(qs, upgradeQlo, availableMlModels, dbItem, progress, token, queryInfo); if (result.resultType !== messages.QueryResultType.SUCCESS) { const message = result.message || 'Failed to run query'; void logger.log(message); From ce86ec17a20d55b2c38bbbae8046c0fca7851807 Mon Sep 17 00:00:00 2001 From: Edoardo Pirovano Date: Mon, 14 Mar 2022 10:54:05 +0000 Subject: [PATCH 2/3] Address review comments from `@aeisenberg` --- extensions/ql-vscode/src/query-history.ts | 25 +++++++++++++---------- extensions/ql-vscode/src/query-results.ts | 8 ++++++++ extensions/ql-vscode/src/run-queries.ts | 2 +- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/extensions/ql-vscode/src/query-history.ts b/extensions/ql-vscode/src/query-history.ts index 22a14fa9ac4..ae207fa10f2 100644 --- a/extensions/ql-vscode/src/query-history.ts +++ b/extensions/ql-vscode/src/query-history.ts @@ -767,13 +767,15 @@ export class QueryHistoryManager extends DisposableObject { singleItem: QueryHistoryInfo, multiSelect: QueryHistoryInfo[] ) { - // Local queries only - if (!this.assertSingleQuery(multiSelect) || singleItem?.t !== 'local') { + const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect); + + // Only applicable to an individual local query + if (!this.assertSingleQuery(finalMultiSelect) || !finalSingleItem || finalSingleItem.t !== 'local') { return; } - if (singleItem.evalLogLocation) { - await this.tryOpenExternalFile(singleItem.evalLogLocation); + if (finalSingleItem.evalLogLocation) { + await this.tryOpenExternalFile(finalSingleItem.evalLogLocation); } else { this.warnNoEvalLog(); } @@ -783,17 +785,18 @@ export class QueryHistoryManager extends DisposableObject { singleItem: QueryHistoryInfo, multiSelect: QueryHistoryInfo[] ) { - // Local queries only - if (!this.assertSingleQuery(multiSelect) || singleItem?.t !== 'local') { + const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect); + + // Only applicable to an individual local query + if (!this.assertSingleQuery(finalMultiSelect) || !finalSingleItem || finalSingleItem.t !== 'local') { return; } - if (singleItem.evalLogLocation) { - const summaryLocation = singleItem.evalLogLocation + '.summary'; - if (!fs.existsSync(summaryLocation)) { - await this.qs.cliServer.generateLogSummary(singleItem.evalLogLocation, summaryLocation); + if (finalSingleItem.evalLogLocation) { + if (!fs.existsSync(finalSingleItem.evalLogSummaryLocation)) { + await this.qs.cliServer.generateLogSummary(finalSingleItem.evalLogLocation, finalSingleItem.evalLogSummaryLocation); } - await this.tryOpenExternalFile(summaryLocation); + await this.tryOpenExternalFile(finalSingleItem.evalLogSummaryLocation); } else { this.warnNoEvalLog(); } diff --git a/extensions/ql-vscode/src/query-results.ts b/extensions/ql-vscode/src/query-results.ts index 7a2426a13f6..f6d10e37565 100644 --- a/extensions/ql-vscode/src/query-results.ts +++ b/extensions/ql-vscode/src/query-results.ts @@ -312,6 +312,14 @@ export class LocalQueryInfo { } } + /** + * Return the location of a query's evaluator log summary. This file may not exist yet, + * in which case it can be created by invoking `codeql generate log-summary`. + */ + get evalLogSummaryLocation(): string { + return this.evalLogLocation + '.summary'; + } + get completed(): boolean { return !!this.completedQuery; } diff --git a/extensions/ql-vscode/src/run-queries.ts b/extensions/ql-vscode/src/run-queries.ts index 6c47abb9f61..d41eaad6981 100644 --- a/extensions/ql-vscode/src/run-queries.ts +++ b/extensions/ql-vscode/src/run-queries.ts @@ -676,7 +676,7 @@ export async function compileAndRunQueryAgainstDatabase( progress: ProgressCallback, token: CancellationToken, templates?: messages.TemplateDefinitions, - queryInfo?: LocalQueryInfo, + queryInfo?: LocalQueryInfo, // May be omitted for queries not initiated by the user. If omitted we won't create a structured log for the query. ): Promise { if (!dbItem.contents || !dbItem.contents.dbSchemeUri) { throw new Error(`Database ${dbItem.databaseUri} does not have a CodeQL database scheme.`); From 9db40c22d5213c236aae705cf744b460514f81d1 Mon Sep 17 00:00:00 2001 From: Angela P Wen Date: Mon, 28 Mar 2022 15:34:33 -0700 Subject: [PATCH 3/3] Gate show eval log and summary commands behind CLI v2.8.4 (#1243) --- extensions/ql-vscode/package.json | 4 ++-- extensions/ql-vscode/src/cli.ts | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 06f592626bf..2046f8ee89e 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -736,12 +736,12 @@ { "command": "codeQLQueryHistory.showEvalLog", "group": "9_qlCommands", - "when": "viewItem == rawResultsItem || viewItem == interpretedResultsItem || viewItem == cancelledResultsItem" + "when": "codeql.supportsEvalLog && (viewItem == rawResultsItem || viewItem == interpretedResultsItem || viewItem == cancelledResultsItem)" }, { "command": "codeQLQueryHistory.showEvalLogSummary", "group": "9_qlCommands", - "when": "viewItem == rawResultsItem || viewItem == interpretedResultsItem || viewItem == cancelledResultsItem" + "when": "codeql.supportsEvalLog && (viewItem == rawResultsItem || viewItem == interpretedResultsItem || viewItem == cancelledResultsItem)" }, { "command": "codeQLQueryHistory.showQueryText", diff --git a/extensions/ql-vscode/src/cli.ts b/extensions/ql-vscode/src/cli.ts index f8e7d654008..7ee7d5e8712 100644 --- a/extensions/ql-vscode/src/cli.ts +++ b/extensions/ql-vscode/src/cli.ts @@ -8,7 +8,7 @@ import { Readable } from 'stream'; import { StringDecoder } from 'string_decoder'; import * as tk from 'tree-kill'; import { promisify } from 'util'; -import { CancellationToken, Disposable, Uri } from 'vscode'; +import { CancellationToken, commands, Disposable, Uri } from 'vscode'; import { BQRSInfo, DecodedBqrsChunk } from './pure/bqrs-cli-types'; import { CliConfig } from './config'; @@ -957,6 +957,10 @@ export class CodeQLCliServer implements Disposable { public async getVersion() { if (!this._version) { this._version = await this.refreshVersion(); + // this._version is only undefined upon config change, so we reset CLI-based context key only when necessary. + await commands.executeCommand( + 'setContext', 'codeql.supportsEvalLog', await this.cliConstraints.supportsPerQueryEvalLog() + ); } return this._version; }