diff --git a/extensions/ql-vscode/CHANGELOG.md b/extensions/ql-vscode/CHANGELOG.md index ad0225ff29e..6e8cad8b227 100644 --- a/extensions/ql-vscode/CHANGELOG.md +++ b/extensions/ql-vscode/CHANGELOG.md @@ -2,7 +2,8 @@ ## [UNRELEASED] -- Fix the _CodeQL: Open Referenced File_ command for Windows systems. [#979](https://github.com/github/vscode-codeql/pull/979) +- Fix the _CodeQL: Open Referenced File_ command for Windows systems. [#979](https://github.com/github/vscode-codeql/pull/979) +- Support large SARIF results files (>4GB) without crashing VS Code. [#1004](https://github.com/github/vscode-codeql/pull/1004) - Fix a bug that shows 'Set current database' when hovering over the currently selected database in the databases view. [#976](https://github.com/github/vscode-codeql/pull/976) - Fix a bug with importing large databases. Databases over 4GB can now be imported directly from LGTM or from a zip file. This functionality is only available when using CodeQL CLI version 2.6.0 or later. [#971](https://github.com/github/vscode-codeql/pull/971) - Replace certain control codes (`U+0000` - `U+001F`) with their corresponding control labels (`U+2400` - `U+241F`) in the results view. [#963](https://github.com/github/vscode-codeql/pull/963) diff --git a/extensions/ql-vscode/package-lock.json b/extensions/ql-vscode/package-lock.json index a055803e1c7..6f8e5814362 100644 --- a/extensions/ql-vscode/package-lock.json +++ b/extensions/ql-vscode/package-lock.json @@ -21,6 +21,9 @@ "react": "^16.8.6", "react-dom": "^16.8.6", "semver": "~7.3.2", + "stream": "^0.0.2", + "stream-chain": "~2.2.4", + "stream-json": "~1.7.3", "tmp": "^0.1.0", "tmp-promise": "~3.0.2", "tree-kill": "~1.2.2", @@ -55,6 +58,8 @@ "@types/semver": "~7.2.0", "@types/sinon": "~7.5.2", "@types/sinon-chai": "~3.2.3", + "@types/stream-chain": "~2.0.1", + "@types/stream-json": "~1.7.1", "@types/through2": "^2.0.36", "@types/tmp": "^0.1.0", "@types/unzipper": "~0.10.1", @@ -852,6 +857,25 @@ "integrity": "sha512-K5K+yml8LTo9bWJI/rECfIPrGgxdpeNbj+d53lwN4QjW1MCwlkhUms+gtdzigTeUyBr09+u8BwOIY3MXvHdcsA==", "dev": true }, + "node_modules/@types/stream-chain": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@types/stream-chain/-/stream-chain-2.0.1.tgz", + "integrity": "sha512-D+Id9XpcBpampptkegH7WMsEk6fUdf9LlCIX7UhLydILsqDin4L0QT7ryJR0oycwC7OqohIzdfcMHVZ34ezNGg==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/stream-json": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/@types/stream-json/-/stream-json-1.7.1.tgz", + "integrity": "sha512-BNIK/ix6iJvWvoXbDVVJhw5LNG1wie/rXcUo7jw4hBqY3FhIrg0e+RMXFN5UreKclBIStl9FDEHNSDLuuQ9/MQ==", + "dev": true, + "dependencies": { + "@types/node": "*", + "@types/stream-chain": "*" + } + }, "node_modules/@types/tapable": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/@types/tapable/-/tapable-1.0.6.tgz", @@ -3543,6 +3567,11 @@ "integrity": "sha512-EGuiJW4yBPOTj2NtWGZcX93ZE8IGj33HJAx4d3ouE2zOfW2trbWU+t1e0yzLr1qQIw81++txbM3BH52QwSRE6Q==", "dev": true }, + "node_modules/emitter-component": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/emitter-component/-/emitter-component-1.1.1.tgz", + "integrity": "sha1-Bl4tvtaVm/RwZ57avq95gdEAOrY=" + }, "node_modules/emitter-listener": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/emitter-listener/-/emitter-listener-1.1.2.tgz", @@ -4715,6 +4744,7 @@ "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz", "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", "dev": true, + "hasInstallScript": true, "optional": true, "os": [ "darwin" @@ -9720,12 +9750,33 @@ "node": ">=0.10.0" } }, + "node_modules/stream": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stream/-/stream-0.0.2.tgz", + "integrity": "sha1-f1Nj8Ff2WSxVlfALyAon9c7B8O8=", + "dependencies": { + "emitter-component": "^1.1.1" + } + }, + "node_modules/stream-chain": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.4.tgz", + "integrity": "sha512-9lsl3YM53V5N/I1C2uJtc3Kavyi3kNYN83VkKb/bMWRk7D9imiFyUPYa0PoZbLohSVOX1mYE9YsmwObZUsth6Q==" + }, "node_modules/stream-exhaust": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/stream-exhaust/-/stream-exhaust-1.0.2.tgz", "integrity": "sha512-b/qaq/GlBK5xaq1yrK9/zFcyRSTNxmcZwFLGSTG0mXgZl/4Z6GgiyYOXOvY7N3eEvFRAG1bkDRz5EPGSvPYQlw==", "dev": true }, + "node_modules/stream-json": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.7.3.tgz", + "integrity": "sha512-Y6dXn9KKWSwxOqnvHGcdZy1PK+J+7alBwHCeU3W9oRqm4ilLRA0XSPmd1tWwhg7tv9EIxJTMWh7KF15tYelKJg==", + "dependencies": { + "stream-chain": "^2.2.4" + } + }, "node_modules/stream-shift": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", @@ -12526,6 +12577,25 @@ "integrity": "sha512-K5K+yml8LTo9bWJI/rECfIPrGgxdpeNbj+d53lwN4QjW1MCwlkhUms+gtdzigTeUyBr09+u8BwOIY3MXvHdcsA==", "dev": true }, + "@types/stream-chain": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@types/stream-chain/-/stream-chain-2.0.1.tgz", + "integrity": "sha512-D+Id9XpcBpampptkegH7WMsEk6fUdf9LlCIX7UhLydILsqDin4L0QT7ryJR0oycwC7OqohIzdfcMHVZ34ezNGg==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "@types/stream-json": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/@types/stream-json/-/stream-json-1.7.1.tgz", + "integrity": "sha512-BNIK/ix6iJvWvoXbDVVJhw5LNG1wie/rXcUo7jw4hBqY3FhIrg0e+RMXFN5UreKclBIStl9FDEHNSDLuuQ9/MQ==", + "dev": true, + "requires": { + "@types/node": "*", + "@types/stream-chain": "*" + } + }, "@types/tapable": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/@types/tapable/-/tapable-1.0.6.tgz", @@ -14728,6 +14798,11 @@ "integrity": "sha512-EGuiJW4yBPOTj2NtWGZcX93ZE8IGj33HJAx4d3ouE2zOfW2trbWU+t1e0yzLr1qQIw81++txbM3BH52QwSRE6Q==", "dev": true }, + "emitter-component": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/emitter-component/-/emitter-component-1.1.1.tgz", + "integrity": "sha1-Bl4tvtaVm/RwZ57avq95gdEAOrY=" + }, "emitter-listener": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/emitter-listener/-/emitter-listener-1.1.2.tgz", @@ -19683,12 +19758,33 @@ } } }, + "stream": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stream/-/stream-0.0.2.tgz", + "integrity": "sha1-f1Nj8Ff2WSxVlfALyAon9c7B8O8=", + "requires": { + "emitter-component": "^1.1.1" + } + }, + "stream-chain": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.4.tgz", + "integrity": "sha512-9lsl3YM53V5N/I1C2uJtc3Kavyi3kNYN83VkKb/bMWRk7D9imiFyUPYa0PoZbLohSVOX1mYE9YsmwObZUsth6Q==" + }, "stream-exhaust": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/stream-exhaust/-/stream-exhaust-1.0.2.tgz", "integrity": "sha512-b/qaq/GlBK5xaq1yrK9/zFcyRSTNxmcZwFLGSTG0mXgZl/4Z6GgiyYOXOvY7N3eEvFRAG1bkDRz5EPGSvPYQlw==", "dev": true }, + "stream-json": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.7.3.tgz", + "integrity": "sha512-Y6dXn9KKWSwxOqnvHGcdZy1PK+J+7alBwHCeU3W9oRqm4ilLRA0XSPmd1tWwhg7tv9EIxJTMWh7KF15tYelKJg==", + "requires": { + "stream-chain": "^2.2.4" + } + }, "stream-shift": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 9028f1e1fe4..4d5d36bd556 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -969,6 +969,9 @@ "react": "^16.8.6", "react-dom": "^16.8.6", "semver": "~7.3.2", + "stream": "^0.0.2", + "stream-chain": "~2.2.4", + "stream-json": "~1.7.3", "tmp": "^0.1.0", "tmp-promise": "~3.0.2", "tree-kill": "~1.2.2", @@ -1003,6 +1006,8 @@ "@types/semver": "~7.2.0", "@types/sinon": "~7.5.2", "@types/sinon-chai": "~3.2.3", + "@types/stream-chain": "~2.0.1", + "@types/stream-json": "~1.7.1", "@types/through2": "^2.0.36", "@types/tmp": "^0.1.0", "@types/unzipper": "~0.10.1", diff --git a/extensions/ql-vscode/src/cli.ts b/extensions/ql-vscode/src/cli.ts index 8c1515a28a3..a73299fcb37 100644 --- a/extensions/ql-vscode/src/cli.ts +++ b/extensions/ql-vscode/src/cli.ts @@ -1,6 +1,5 @@ import * as cpp from 'child-process-promise'; import * as child_process from 'child_process'; -import * as fs from 'fs-extra'; import * as path from 'path'; import * as sarif from 'sarif'; import { SemVer } from 'semver'; @@ -17,6 +16,7 @@ import { assertNever } from './pure/helpers-pure'; import { QueryMetadata, SortDirection } from './pure/interface-types'; import { Logger, ProgressReporter } from './logging'; import { CompilationMessage } from './pure/messages'; +import { sarifParser } from './sarif-parser'; import { dbSchemeToLanguage } from './helpers'; /** @@ -682,22 +682,7 @@ export class CodeQLCliServer implements Disposable { async interpretBqrs(metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise { await this.runInterpretCommand(SARIF_FORMAT, metadata, resultsPath, interpretedResultsPath, sourceInfo); - - let output: string; - try { - output = await fs.readFile(interpretedResultsPath, 'utf8'); - } catch (e) { - const rawMessage = e.stderr || e.message; - const errorMessage = rawMessage.startsWith('Cannot create a string') - ? `SARIF too large. ${rawMessage}` - : rawMessage; - throw new Error(`Reading output of interpretation failed: ${errorMessage}`); - } - try { - return JSON.parse(output) as sarif.Log; - } catch (err) { - throw new Error(`Parsing output of interpretation failed: ${err.stderr || err}`); - } + return await sarifParser(interpretedResultsPath); } async generateResultsCsv(metadata: QueryMetadata, resultsPath: string, csvPath: string, sourceInfo?: SourceInfo): Promise { @@ -1143,7 +1128,7 @@ export class CliVersionConstraint { /** * CLI version where database registration was introduced - */ + */ public static CLI_VERSION_WITH_DB_REGISTRATION = new SemVer('2.4.1'); /** diff --git a/extensions/ql-vscode/src/sarif-parser.ts b/extensions/ql-vscode/src/sarif-parser.ts new file mode 100644 index 00000000000..eed080b4639 --- /dev/null +++ b/extensions/ql-vscode/src/sarif-parser.ts @@ -0,0 +1,48 @@ +import * as Sarif from 'sarif'; +import * as fs from 'fs-extra'; +import { parser } from 'stream-json'; +import { pick } from 'stream-json/filters/Pick'; +import Assembler = require('stream-json/Assembler'); +import { chain } from 'stream-chain'; + +const DUMMY_TOOL : Sarif.Tool = {driver: {name: ''}}; + +export async function sarifParser(interpretedResultsPath: string) : Promise { + try { + // Parse the SARIF file into token streams, filtering out only the results array. + const p = parser(); + const pipeline = chain([ + fs.createReadStream(interpretedResultsPath), + p, + pick({filter: 'runs.0.results'}) + ]); + + // Creates JavaScript objects from the token stream + const asm = Assembler.connectTo(pipeline); + + // Returns a constructed Log object with the results or an empty array if no results were found. + // If the parser fails for any reason, it will reject the promise. + return await new Promise((resolve, reject) => { + pipeline.on('error', (error) => { + reject(error); + }); + + asm.on('done', (asm) => { + + const log : Sarif.Log = { + version: '2.1.0', + runs: [ + { + tool: DUMMY_TOOL, + results: asm.current ?? [] + } + ] + }; + + resolve(log); + }); + }); + } catch (err) { + throw new Error(`Parsing output of interpretation failed: ${err.stderr || err}`); + } +} \ No newline at end of file diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/data/sarif/emptyResultsSarif.sarif b/extensions/ql-vscode/src/vscode-tests/no-workspace/data/sarif/emptyResultsSarif.sarif new file mode 100644 index 00000000000..b61d130f827 --- /dev/null +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/data/sarif/emptyResultsSarif.sarif @@ -0,0 +1,34 @@ +{ + "version": "2.1.0", + "$schema": "http://json.schemastore.org/sarif-2.1.0-rtm.4", + "runs": [ + { + "tool": { + "driver": { + "name": "ESLint", + "informationUri": "https://eslint.org", + "rules": [ + { + "id": "no-unused-vars", + "shortDescription": { + "text": "disallow unused variables" + }, + "helpUri": "https://eslint.org/docs/rules/no-unused-vars", + "properties": { + "category": "Variables" + } + } + ] + } + }, + "artifacts": [ + { + "location": { + "uri": "file:///C:/dev/sarif/sarif-tutorials/samples/Introduction/simple-example.js" + } + } + ], + "results": [] + } + ] +} \ No newline at end of file diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/data/sarif/invalidSarif.sarif b/extensions/ql-vscode/src/vscode-tests/no-workspace/data/sarif/invalidSarif.sarif new file mode 100644 index 00000000000..a5ec27484ad --- /dev/null +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/data/sarif/invalidSarif.sarif @@ -0,0 +1,33 @@ +{ + "version": "2.1.0", + "$schema": "http://json.schemastore.org/sarif-2.1.0-rtm.4", + "runs": [ + { + "tool": { + "driver": { + "name": "ESLint", + "informationUri": "https://eslint.org", + "rules": [ + { + "id": "no-unused-vars", + "shortDescription": { + "text": "disallow unused variables" + }, + "helpUri": "https://eslint.org/docs/rules/no-unused-vars", + "properties": { + "category": "Variables" + } + } + ] + } + }, + "artifacts": [ + { + "location": { + "uri": "file:///C:/dev/sarif/sarif-tutorials/samples/Introduction/simple-example.js" + } + } + ] + } + ] +} \ No newline at end of file diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/data/sarif/validSarif.sarif b/extensions/ql-vscode/src/vscode-tests/no-workspace/data/sarif/validSarif.sarif new file mode 100644 index 00000000000..2d2fa5cacfa --- /dev/null +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/data/sarif/validSarif.sarif @@ -0,0 +1,57 @@ +{ + "version": "2.1.0", + "$schema": "http://json.schemastore.org/sarif-2.1.0-rtm.4", + "runs": [ + { + "tool": { + "driver": { + "name": "ESLint", + "informationUri": "https://eslint.org", + "rules": [ + { + "id": "no-unused-vars", + "shortDescription": { + "text": "disallow unused variables" + }, + "helpUri": "https://eslint.org/docs/rules/no-unused-vars", + "properties": { + "category": "Variables" + } + } + ] + } + }, + "artifacts": [ + { + "location": { + "uri": "file:///C:/dev/sarif/sarif-tutorials/samples/Introduction/simple-example.js" + } + } + ], + "results": [ + { + "level": "error", + "message": { + "text": "'x' is assigned a value but never used." + }, + "locations": [ + { + "physicalLocation": { + "artifactLocation": { + "uri": "file:///C:/dev/sarif/sarif-tutorials/samples/Introduction/simple-example.js", + "index": 0 + }, + "region": { + "startLine": 1, + "startColumn": 5 + } + } + } + ], + "ruleId": "no-unused-vars", + "ruleIndex": 0 + } + ] + } + ] +} \ No newline at end of file diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/sarifParser.test.ts b/extensions/ql-vscode/src/vscode-tests/no-workspace/sarifParser.test.ts new file mode 100644 index 00000000000..f9b2fedf36e --- /dev/null +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/sarifParser.test.ts @@ -0,0 +1,25 @@ +import * as path from 'path'; +import * as chai from 'chai'; +import * as chaiAsPromised from 'chai-as-promised'; + +import { sarifParser } from '../../sarif-parser'; + +chai.use(chaiAsPromised); +const expect = chai.expect; + +describe.only('sarif parser', function() { + const sarifDir = path.join(__dirname, 'data/sarif'); + it('should parse a valid SARIF file', async () => { + const result = await sarifParser(path.join(sarifDir, 'validSarif.sarif')); + expect(result.version).to.exist; + expect(result.runs).to.exist; + expect(result.runs[0].tool).to.exist; + expect(result.runs[0].tool.driver).to.exist; + expect(result.runs.length).to.be.at.least(1); + }); + + it('should return an empty array if there are no results', async () => { + const result = await sarifParser(path.join(sarifDir, 'emptyResultsSarif.sarif')); + expect(result.runs[0].results).to.be.empty; + }); +}); \ No newline at end of file diff --git a/extensions/ql-vscode/test/pure-tests/sarif-utils.test.ts b/extensions/ql-vscode/test/pure-tests/sarif-utils.test.ts index 6976217a73d..39e1be0b812 100644 --- a/extensions/ql-vscode/test/pure-tests/sarif-utils.test.ts +++ b/extensions/ql-vscode/test/pure-tests/sarif-utils.test.ts @@ -6,11 +6,12 @@ import { getPathRelativeToSourceLocationPrefix, parseSarifLocation, parseSarifPlainTextMessage, - unescapeSarifText + unescapeSarifText, } from '../../src/pure/sarif-utils'; describe('parsing sarif', () => { + it('should be able to parse a simple message from the spec', async function() { const message = 'Tainted data was used. The data came from [here](3).'; const results = parseSarifPlainTextMessage(message);