diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 87788fe8dea..c4000edef50 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -516,6 +516,10 @@ "title": "Add new list", "icon": "$(new-folder)" }, + { + "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch", + "title": "Add repositories with GitHub Code Search" + }, { "command": "codeQLVariantAnalysisRepositories.setSelectedItem", "title": "Select" @@ -961,6 +965,11 @@ "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canBeOpenedOnGitHub/", "group": "2_qlContextMenu@1" }, + { + "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch", + "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canImportCodeSearch/", + "group": "2_qlContextMenu@1" + }, { "command": "codeQLDatabases.setCurrentDatabase", "group": "inline", @@ -1297,6 +1306,10 @@ "command": "codeQLVariantAnalysisRepositories.removeItemContextMenu", "when": "false" }, + { + "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch", + "when": "false" + }, { "command": "codeQLDatabases.setCurrentDatabase", "when": "false" diff --git a/extensions/ql-vscode/src/common/commands.ts b/extensions/ql-vscode/src/common/commands.ts index 1b0a677a2e2..74d5560af3f 100644 --- a/extensions/ql-vscode/src/common/commands.ts +++ b/extensions/ql-vscode/src/common/commands.ts @@ -275,6 +275,7 @@ export type DatabasePanelCommands = { "codeQLVariantAnalysisRepositories.openOnGitHubContextMenu": TreeViewContextSingleSelectionCommandFunction; "codeQLVariantAnalysisRepositories.renameItemContextMenu": TreeViewContextSingleSelectionCommandFunction; "codeQLVariantAnalysisRepositories.removeItemContextMenu": TreeViewContextSingleSelectionCommandFunction; + "codeQLVariantAnalysisRepositories.importFromCodeSearch": TreeViewContextSingleSelectionCommandFunction; }; export type AstCfgCommands = { diff --git a/extensions/ql-vscode/src/databases/config/db-config-store.ts b/extensions/ql-vscode/src/databases/config/db-config-store.ts index 3013d164f50..bc1f5771b7d 100644 --- a/extensions/ql-vscode/src/databases/config/db-config-store.ts +++ b/extensions/ql-vscode/src/databases/config/db-config-store.ts @@ -145,10 +145,46 @@ export class DbConfigStore extends DisposableObject { await this.writeConfig(config); } + /** + * Adds a list of remote repositories to an existing repository list and removes duplicates. + * @returns a list of repositories that were not added because the list reached 1000 entries. + */ + public async addRemoteReposToList( + repoNwoList: string[], + parentList: string, + ): Promise { + if (!this.config) { + throw Error("Cannot add variant analysis repos if config is not loaded"); + } + + const config = cloneDbConfig(this.config); + const parent = config.databases.variantAnalysis.repositoryLists.find( + (list) => list.name === parentList, + ); + if (!parent) { + throw Error(`Cannot find parent list '${parentList}'`); + } + + // Remove duplicates from the list of repositories. + const newRepositoriesList = [ + ...new Set([...parent.repositories, ...repoNwoList]), + ]; + + parent.repositories = newRepositoriesList.slice(0, 1000); + const truncatedRepositories = newRepositoriesList.slice(1000); + + await this.writeConfig(config); + return truncatedRepositories; + } + + /** + * Adds one remote repository + * @returns either nothing, or, if a parentList is given AND the number of repos on that list reaches 1000 returns the repo that was not added. + */ public async addRemoteRepo( repoNwo: string, parentList?: string, - ): Promise { + ): Promise { if (!this.config) { throw Error("Cannot add variant analysis repo if config is not loaded"); } @@ -163,6 +199,7 @@ export class DbConfigStore extends DisposableObject { ); } + const truncatedRepositories = []; const config = cloneDbConfig(this.config); if (parentList) { const parent = config.databases.variantAnalysis.repositoryLists.find( @@ -171,12 +208,15 @@ export class DbConfigStore extends DisposableObject { if (!parent) { throw Error(`Cannot find parent list '${parentList}'`); } else { - parent.repositories.push(repoNwo); + const newRepositories = [...parent.repositories, repoNwo]; + parent.repositories = newRepositories.slice(0, 1000); + truncatedRepositories.push(...newRepositories.slice(1000)); } } else { config.databases.variantAnalysis.repositories.push(repoNwo); } await this.writeConfig(config); + return truncatedRepositories; } public async addRemoteOwner(owner: string): Promise { diff --git a/extensions/ql-vscode/src/databases/db-manager.ts b/extensions/ql-vscode/src/databases/db-manager.ts index cc85bccc01e..5c1a156d990 100644 --- a/extensions/ql-vscode/src/databases/db-manager.ts +++ b/extensions/ql-vscode/src/databases/db-manager.ts @@ -96,8 +96,15 @@ export class DbManager { public async addNewRemoteRepo( nwo: string, parentList?: string, - ): Promise { - await this.dbConfigStore.addRemoteRepo(nwo, parentList); + ): Promise { + return await this.dbConfigStore.addRemoteRepo(nwo, parentList); + } + + public async addNewRemoteReposToList( + nwoList: string[], + parentList: string, + ): Promise { + return await this.dbConfigStore.addRemoteReposToList(nwoList, parentList); } public async addNewRemoteOwner(owner: string): Promise { diff --git a/extensions/ql-vscode/src/databases/ui/db-panel.ts b/extensions/ql-vscode/src/databases/ui/db-panel.ts index 16abf9f5073..9172b28220b 100644 --- a/extensions/ql-vscode/src/databases/ui/db-panel.ts +++ b/extensions/ql-vscode/src/databases/ui/db-panel.ts @@ -1,4 +1,5 @@ import { + ProgressLocation, QuickPickItem, TreeView, TreeViewExpansionEvent, @@ -13,7 +14,10 @@ import { getOwnerFromGitHubUrl, isValidGitHubOwner, } from "../../common/github-url-identifier-helper"; -import { showAndLogErrorMessage } from "../../helpers"; +import { + showAndLogErrorMessage, + showAndLogInformationMessage, +} from "../../helpers"; import { DisposableObject } from "../../pure/disposable-object"; import { DbItem, @@ -32,6 +36,8 @@ import { getControllerRepo } from "../../variant-analysis/run-remote-query"; import { getErrorMessage } from "../../pure/helpers-pure"; import { DatabasePanelCommands } from "../../common/commands"; import { App } from "../../common/app"; +import { getCodeSearchRepositories } from "../../variant-analysis/gh-api/gh-api-client"; +import { QueryLanguage } from "../../common/query-language"; export interface RemoteDatabaseQuickPickItem extends QuickPickItem { remoteDatabaseKind: string; @@ -41,6 +47,10 @@ export interface AddListQuickPickItem extends QuickPickItem { databaseKind: DbListKind; } +export interface CodeSearchQuickPickItem extends QuickPickItem { + language: string; +} + export class DbPanel extends DisposableObject { private readonly dataProvider: DbTreeDataProvider; private readonly treeView: TreeView; @@ -93,6 +103,8 @@ export class DbPanel extends DisposableObject { this.renameItem.bind(this), "codeQLVariantAnalysisRepositories.removeItemContextMenu": this.removeItem.bind(this), + "codeQLVariantAnalysisRepositories.importFromCodeSearch": + this.importFromCodeSearch.bind(this), }; } @@ -171,7 +183,14 @@ export class DbPanel extends DisposableObject { return; } - await this.dbManager.addNewRemoteRepo(nwo, parentList); + const truncatedRepositories = await this.dbManager.addNewRemoteRepo( + nwo, + parentList, + ); + + if (parentList) { + this.reportAnyTruncatedRepos(truncatedRepositories, parentList); + } } private async addNewRemoteOwner(): Promise { @@ -323,6 +342,89 @@ export class DbPanel extends DisposableObject { await this.dbManager.removeDbItem(treeViewItem.dbItem); } + private async importFromCodeSearch( + treeViewItem: DbTreeViewItem, + ): Promise { + if (treeViewItem.dbItem?.kind !== DbItemKind.RemoteUserDefinedList) { + throw new Error("Please select a valid list to add code search results."); + } + + const listName = treeViewItem.dbItem.listName; + + const languageQuickPickItems: CodeSearchQuickPickItem[] = Object.values( + QueryLanguage, + ).map((language) => ({ + label: language.toString(), + alwaysShow: true, + language: language.toString(), + })); + + const codeSearchLanguage = + await window.showQuickPick( + languageQuickPickItems, + { + title: "Select a language for your search", + placeHolder: "Select an option", + ignoreFocusOut: true, + }, + ); + if (!codeSearchLanguage) { + return; + } + + const codeSearchQuery = await window.showInputBox({ + title: "GitHub Code Search", + prompt: + "Use [GitHub's Code Search syntax](https://docs.github.com/en/search-github/github-code-search/understanding-github-code-search-syntax), including code qualifiers, regular expressions, and boolean operations, to search for repositories.", + placeHolder: "org:github", + }); + if (codeSearchQuery === undefined || codeSearchQuery === "") { + return; + } + + void window.withProgress( + { + location: ProgressLocation.Notification, + title: "Searching for repositories... This might take a while", + cancellable: true, + }, + async (progress, token) => { + progress.report({ increment: 10 }); + + const repositories = await getCodeSearchRepositories( + this.app.credentials, + `${codeSearchQuery} language:${codeSearchLanguage.language}`, + progress, + token, + ); + + token.onCancellationRequested(() => { + void showAndLogInformationMessage("Code search cancelled"); + return; + }); + + progress.report({ increment: 10, message: "Processing results..." }); + + const truncatedRepositories = + await this.dbManager.addNewRemoteReposToList(repositories, listName); + this.reportAnyTruncatedRepos(truncatedRepositories, listName); + }, + ); + } + + private reportAnyTruncatedRepos( + truncatedRepositories: string[], + listName: string, + ) { + if (truncatedRepositories.length > 0) { + void showAndLogErrorMessage( + `Some repositories were not added to '${listName}' because a list can only have 1000 entries. Excluded repositories: ${truncatedRepositories.join( + ", ", + )}`, + ); + } + } + private async onDidCollapseElement( event: TreeViewExpansionEvent, ): Promise { diff --git a/extensions/ql-vscode/src/databases/ui/db-tree-view-item-action.ts b/extensions/ql-vscode/src/databases/ui/db-tree-view-item-action.ts index 2755934f772..315b12c8b76 100644 --- a/extensions/ql-vscode/src/databases/ui/db-tree-view-item-action.ts +++ b/extensions/ql-vscode/src/databases/ui/db-tree-view-item-action.ts @@ -4,7 +4,8 @@ export type DbTreeViewItemAction = | "canBeSelected" | "canBeRemoved" | "canBeRenamed" - | "canBeOpenedOnGitHub"; + | "canBeOpenedOnGitHub" + | "canImportCodeSearch"; export function getDbItemActions(dbItem: DbItem): DbTreeViewItemAction[] { const actions: DbTreeViewItemAction[] = []; @@ -21,7 +22,9 @@ export function getDbItemActions(dbItem: DbItem): DbTreeViewItemAction[] { if (canBeOpenedOnGitHub(dbItem)) { actions.push("canBeOpenedOnGitHub"); } - + if (canImportCodeSearch(dbItem)) { + actions.push("canImportCodeSearch"); + } return actions; } @@ -60,6 +63,10 @@ function canBeOpenedOnGitHub(dbItem: DbItem): boolean { return dbItemKindsThatCanBeOpenedOnGitHub.includes(dbItem.kind); } +function canImportCodeSearch(dbItem: DbItem): boolean { + return DbItemKind.RemoteUserDefinedList === dbItem.kind; +} + export function getGitHubUrl(dbItem: DbItem): string | undefined { switch (dbItem.kind) { case DbItemKind.RemoteOwner: diff --git a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts index 00bbafde926..0e8d68d1f7a 100644 --- a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts +++ b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts @@ -7,6 +7,43 @@ import { VariantAnalysisSubmissionRequest, } from "./variant-analysis"; import { Repository } from "./repository"; +import { Progress } from "vscode"; +import { CancellationToken } from "vscode-jsonrpc"; + +export async function getCodeSearchRepositories( + credentials: Credentials, + query: string, + progress: Progress<{ + message?: string | undefined; + increment?: number | undefined; + }>, + token: CancellationToken, +): Promise { + let nwos: string[] = []; + const octokit = await credentials.getOctokit(); + for await (const response of octokit.paginate.iterator( + octokit.rest.search.repos, + { + q: query, + per_page: 100, + }, + )) { + nwos.push(...response.data.map((item) => item.full_name)); + // calculate progress bar: 80% of the progress bar is used for the code search + const totalNumberOfRequests = Math.ceil(response.data.total_count / 100); + // Since we have a maximum 10 of requests, we use a fixed increment whenever the totalNumberOfRequests is greater than 10 + const increment = + totalNumberOfRequests < 10 ? 80 / totalNumberOfRequests : 8; + progress.report({ increment }); + + if (token.isCancellationRequested) { + nwos = []; + break; + } + } + + return [...new Set(nwos)]; +} export async function submitVariantAnalysis( credentials: Credentials, diff --git a/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts b/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts index ca4b7ead6a7..b7a3fd99746 100644 --- a/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts +++ b/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts @@ -241,6 +241,113 @@ describe("db config store", () => { configStore.dispose(); }); + it("should add unique remote repositories to the correct list", async () => { + // Initial set up + const dbConfig = createDbConfig({ + remoteLists: [ + { + name: "list1", + repositories: ["owner/repo1"], + }, + ], + }); + + const configStore = await initializeConfig(dbConfig, configPath, app); + expect( + configStore.getConfig().value.databases.variantAnalysis + .repositoryLists[0], + ).toEqual({ + name: "list1", + repositories: ["owner/repo1"], + }); + + // Add + const response = await configStore.addRemoteReposToList( + ["owner/repo1", "owner/repo2"], + "list1", + ); + + // Read the config file + const updatedDbConfig = (await readJSON(configPath)) as DbConfig; + + // Check that the config file has been updated + const updatedRemoteDbs = updatedDbConfig.databases.variantAnalysis; + expect(updatedRemoteDbs.repositories).toHaveLength(0); + expect(updatedRemoteDbs.repositoryLists).toHaveLength(1); + expect(updatedRemoteDbs.repositoryLists[0]).toEqual({ + name: "list1", + repositories: ["owner/repo1", "owner/repo2"], + }); + expect(response).toEqual([]); + + configStore.dispose(); + }); + + it("should add no more than 1000 repositories to a remote list when adding multiple repos", async () => { + // Initial set up + const dbConfig = createDbConfig({ + remoteLists: [ + { + name: "list1", + repositories: [], + }, + ], + }); + + const configStore = await initializeConfig(dbConfig, configPath, app); + + // Add + const response = await configStore.addRemoteReposToList( + [...Array(1001).keys()].map((i) => `owner/db${i}`), + "list1", + ); + + // Read the config file + const updatedDbConfig = (await readJSON(configPath)) as DbConfig; + + // Check that the config file has been updated + const updatedRemoteDbs = updatedDbConfig.databases.variantAnalysis; + expect(updatedRemoteDbs.repositories).toHaveLength(0); + expect(updatedRemoteDbs.repositoryLists).toHaveLength(1); + expect(updatedRemoteDbs.repositoryLists[0].repositories).toHaveLength( + 1000, + ); + expect(response).toEqual(["owner/db1000"]); + + configStore.dispose(); + }); + + it("should add no more than 1000 repositories to a remote list when adding one repo", async () => { + // Initial set up + const dbConfig = createDbConfig({ + remoteLists: [ + { + name: "list1", + repositories: [...Array(1000).keys()].map((i) => `owner/db${i}`), + }, + ], + }); + + const configStore = await initializeConfig(dbConfig, configPath, app); + + // Add + const reponse = await configStore.addRemoteRepo("owner/db1000", "list1"); + + // Read the config file + const updatedDbConfig = (await readJSON(configPath)) as DbConfig; + + // Check that the config file has been updated + const updatedRemoteDbs = updatedDbConfig.databases.variantAnalysis; + expect(updatedRemoteDbs.repositories).toHaveLength(0); + expect(updatedRemoteDbs.repositoryLists).toHaveLength(1); + expect(updatedRemoteDbs.repositoryLists[0].repositories).toHaveLength( + 1000, + ); + expect(reponse).toEqual(["owner/db1000"]); + + configStore.dispose(); + }); + it("should add a remote owner", async () => { // Initial set up const dbConfig = createDbConfig(); diff --git a/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts b/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts index fa49ce2f93a..60c87c3565a 100644 --- a/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts +++ b/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts @@ -88,6 +88,73 @@ describe("db manager", () => { ).toEqual("owner2/repo2"); }); + it("should add new remote repos to a user defined list", async () => { + const dbConfig: DbConfig = createDbConfig({ + remoteLists: [ + { + name: "my-list-1", + repositories: ["owner1/repo1"], + }, + ], + }); + + await saveDbConfig(dbConfig); + + await dbManager.addNewRemoteReposToList(["owner2/repo2"], "my-list-1"); + + const dbConfigFileContents = await readDbConfigDirectly(); + expect( + dbConfigFileContents.databases.variantAnalysis.repositoryLists.length, + ).toBe(1); + + expect( + dbConfigFileContents.databases.variantAnalysis.repositoryLists[0], + ).toEqual({ + name: "my-list-1", + repositories: ["owner1/repo1", "owner2/repo2"], + }); + }); + + it("should return truncated repos when adding multiple repos to a user defined list", async () => { + const dbConfig: DbConfig = createDbConfig({ + remoteLists: [ + { + name: "my-list-1", + repositories: [...Array(1000).keys()].map((i) => `owner/db${i}`), + }, + ], + }); + + await saveDbConfig(dbConfig); + + const response = await dbManager.addNewRemoteReposToList( + ["owner2/repo2"], + "my-list-1", + ); + + expect(response).toEqual(["owner2/repo2"]); + }); + + it("should return truncated repos when adding one repo to a user defined list", async () => { + const dbConfig: DbConfig = createDbConfig({ + remoteLists: [ + { + name: "my-list-1", + repositories: [...Array(1000).keys()].map((i) => `owner/db${i}`), + }, + ], + }); + + await saveDbConfig(dbConfig); + + const response = await dbManager.addNewRemoteRepo( + "owner2/repo2", + "my-list-1", + ); + + expect(response).toEqual(["owner2/repo2"]); + }); + it("should add a new remote repo to a user defined list", async () => { const dbConfig: DbConfig = createDbConfig({ remoteLists: [ diff --git a/extensions/ql-vscode/test/unit-tests/databases/ui/db-tree-view-item-action.test.ts b/extensions/ql-vscode/test/unit-tests/databases/ui/db-tree-view-item-action.test.ts index 036be67d873..48f8cfc8373 100644 --- a/extensions/ql-vscode/test/unit-tests/databases/ui/db-tree-view-item-action.test.ts +++ b/extensions/ql-vscode/test/unit-tests/databases/ui/db-tree-view-item-action.test.ts @@ -62,12 +62,17 @@ describe("getDbItemActions", () => { expect(actions.length).toEqual(0); }); - it("should set canBeSelected, canBeRemoved and canBeRenamed for remote user defined db list", () => { + it("should set canBeSelected, canBeRemoved, canBeRenamed and canImportCodeSearch for remote user defined db list", () => { const dbItem = createRemoteUserDefinedListDbItem(); const actions = getDbItemActions(dbItem); - expect(actions).toEqual(["canBeSelected", "canBeRemoved", "canBeRenamed"]); + expect(actions).toEqual([ + "canBeSelected", + "canBeRemoved", + "canBeRenamed", + "canImportCodeSearch", + ]); }); it("should not set canBeSelected for remote user defined db list that is already selected", () => { diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts index d3e08fdace0..8f8c8689fc4 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts @@ -349,7 +349,12 @@ describe("db panel rendering nodes", () => { expect(item.tooltip).toBeUndefined(); expect(item.iconPath).toBeUndefined(); expect(item.collapsibleState).toBe(TreeItemCollapsibleState.Collapsed); - checkDbItemActions(item, ["canBeSelected", "canBeRenamed", "canBeRemoved"]); + checkDbItemActions(item, [ + "canBeSelected", + "canBeRenamed", + "canBeRemoved", + "canImportCodeSearch", + ]); expect(item.children).toBeTruthy(); expect(item.children.length).toBe(repos.length);