Feed code search results into variant analysis repo lists

extensions/ql-vscode/package.json

@@ -516,6 +516,10 @@
         "title": "Add new list",
         "icon": "$(new-folder)"
       },
+      {
+        "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
+        "title": "Add repositories with GitHub Code Search"
+      },
       {
         "command": "codeQLVariantAnalysisRepositories.setSelectedItem",
         "title": "Select"
@@ -961,6 +965,11 @@
           "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canBeOpenedOnGitHub/",
           "group": "2_qlContextMenu@1"
         },
+        {
+          "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
+          "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canImportCodeSearch/",
+          "group": "2_qlContextMenu@1"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "group": "inline",
@@ -1297,6 +1306,10 @@
           "command": "codeQLVariantAnalysisRepositories.removeItemContextMenu",
           "when": "false"
         },
+        {
+          "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
+          "when": "false"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "when": "false"

extensions/ql-vscode/src/common/commands.ts

@@ -275,6 +275,7 @@ export type DatabasePanelCommands = {
   "codeQLVariantAnalysisRepositories.openOnGitHubContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
   "codeQLVariantAnalysisRepositories.renameItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
   "codeQLVariantAnalysisRepositories.removeItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
+  "codeQLVariantAnalysisRepositories.importFromCodeSearch": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
 };
 
 export type AstCfgCommands = {

extensions/ql-vscode/src/databases/config/db-config-store.ts

@@ -145,10 +145,46 @@ export class DbConfigStore extends DisposableObject {
     await this.writeConfig(config);
   }
 
+  /**
+   * Adds a list of remote repositories to an existing repository list and removes duplicates.
+   * @returns a list of repositories that were not added because the list reached 1000 entries.
+   */
+  public async addRemoteReposToList(
+    repoNwoList: string[],
+    parentList: string,
+  ): Promise<string[]> {
+    if (!this.config) {
+      throw Error("Cannot add variant analysis repos if config is not loaded");
+    }
+
+    const config = cloneDbConfig(this.config);
+    const parent = config.databases.variantAnalysis.repositoryLists.find(
+      (list) => list.name === parentList,
+    );
+    if (!parent) {
+      throw Error(`Cannot find parent list '${parentList}'`);
+    }
+
+    // Remove duplicates from the list of repositories.
+    const newRepositoriesList = [
+      ...new Set([...parent.repositories, ...repoNwoList]),
+    ];
+
+    parent.repositories = newRepositoriesList.slice(0, 1000);
+    const truncatedRepositories = newRepositoriesList.slice(1000);
+
+    await this.writeConfig(config);
+    return truncatedRepositories;
+  }
+
+  /**
+   * Adds one remote repository
+   * @returns either nothing, or, if a parentList is given AND the number of repos on that list reaches 1000 returns the repo that was not added.
+   */
   public async addRemoteRepo(
     repoNwo: string,
     parentList?: string,
-  ): Promise<void> {
+  ): Promise<string[]> {
     if (!this.config) {
       throw Error("Cannot add variant analysis repo if config is not loaded");
     }
@@ -163,6 +199,7 @@ export class DbConfigStore extends DisposableObject {
       );
     }
 
+    const truncatedRepositories = [];
     const config = cloneDbConfig(this.config);
     if (parentList) {
       const parent = config.databases.variantAnalysis.repositoryLists.find(
@@ -171,12 +208,15 @@ export class DbConfigStore extends DisposableObject {
       if (!parent) {
         throw Error(`Cannot find parent list '${parentList}'`);
       } else {
-        parent.repositories.push(repoNwo);
+        const newRepositories = [...parent.repositories, repoNwo];
+        parent.repositories = newRepositories.slice(0, 1000);
+        truncatedRepositories.push(...newRepositories.slice(1000));
       }
     } else {
       config.databases.variantAnalysis.repositories.push(repoNwo);
     }
     await this.writeConfig(config);
+    return truncatedRepositories;
   }
 
   public async addRemoteOwner(owner: string): Promise<void> {

extensions/ql-vscode/src/databases/db-manager.ts

@@ -96,8 +96,15 @@ export class DbManager {
   public async addNewRemoteRepo(
     nwo: string,
     parentList?: string,
-  ): Promise<void> {
-    await this.dbConfigStore.addRemoteRepo(nwo, parentList);
+  ): Promise<string[]> {
+    return await this.dbConfigStore.addRemoteRepo(nwo, parentList);
+  }
+
+  public async addNewRemoteReposToList(
+    nwoList: string[],
+    parentList: string,
+  ): Promise<string[]> {
+    return await this.dbConfigStore.addRemoteReposToList(nwoList, parentList);
   }
 
   public async addNewRemoteOwner(owner: string): Promise<void> {

extensions/ql-vscode/src/databases/ui/db-panel.ts

@@ -1,4 +1,5 @@
 import {
+  ProgressLocation,
   QuickPickItem,
   TreeView,
   TreeViewExpansionEvent,
@@ -13,7 +14,10 @@ import {
   getOwnerFromGitHubUrl,
   isValidGitHubOwner,
 } from "../../common/github-url-identifier-helper";
-import { showAndLogErrorMessage } from "../../helpers";
+import {
+  showAndLogErrorMessage,
+  showAndLogInformationMessage,
+} from "../../helpers";
 import { DisposableObject } from "../../pure/disposable-object";
 import {
   DbItem,
@@ -32,6 +36,8 @@ import { getControllerRepo } from "../../variant-analysis/run-remote-query";
 import { getErrorMessage } from "../../pure/helpers-pure";
 import { DatabasePanelCommands } from "../../common/commands";
 import { App } from "../../common/app";
+import { getCodeSearchRepositories } from "../../variant-analysis/gh-api/gh-api-client";
+import { QueryLanguage } from "../../common/query-language";
 
 export interface RemoteDatabaseQuickPickItem extends QuickPickItem {
   remoteDatabaseKind: string;
@@ -41,6 +47,10 @@ export interface AddListQuickPickItem extends QuickPickItem {
   databaseKind: DbListKind;
 }
 
+export interface CodeSearchQuickPickItem extends QuickPickItem {
+  language: string;
+}
+
 export class DbPanel extends DisposableObject {
   private readonly dataProvider: DbTreeDataProvider;
   private readonly treeView: TreeView<DbTreeViewItem>;
@@ -93,6 +103,8 @@ export class DbPanel extends DisposableObject {
         this.renameItem.bind(this),
       "codeQLVariantAnalysisRepositories.removeItemContextMenu":
         this.removeItem.bind(this),
+      "codeQLVariantAnalysisRepositories.importFromCodeSearch":
+        this.importFromCodeSearch.bind(this),
     };
   }
 
@@ -171,7 +183,14 @@ export class DbPanel extends DisposableObject {
       return;
     }
 
-    await this.dbManager.addNewRemoteRepo(nwo, parentList);
+    const truncatedRepositories = await this.dbManager.addNewRemoteRepo(
+      nwo,
+      parentList,
+    );
+
+    if (parentList) {
+      this.reportAnyTruncatedRepos(truncatedRepositories, parentList);
+    }
   }
 
   private async addNewRemoteOwner(): Promise<void> {
@@ -323,6 +342,89 @@ export class DbPanel extends DisposableObject {
     await this.dbManager.removeDbItem(treeViewItem.dbItem);
   }
 
+  private async importFromCodeSearch(
+    treeViewItem: DbTreeViewItem,
+  ): Promise<void> {
+    if (treeViewItem.dbItem?.kind !== DbItemKind.RemoteUserDefinedList) {
+      throw new Error("Please select a valid list to add code search results.");
+    }
+
+    const listName = treeViewItem.dbItem.listName;
+
+    const languageQuickPickItems: CodeSearchQuickPickItem[] = Object.values(
+      QueryLanguage,
+    ).map((language) => ({
+      label: language.toString(),
+      alwaysShow: true,
+      language: language.toString(),
+    }));
+
+    const codeSearchLanguage =
+      await window.showQuickPick<CodeSearchQuickPickItem>(
+        languageQuickPickItems,
+        {
+          title: "Select a language for your search",
+          placeHolder: "Select an option",
+          ignoreFocusOut: true,
+        },
+      );
+    if (!codeSearchLanguage) {
+      return;
+    }
+
+    const codeSearchQuery = await window.showInputBox({
+      title: "GitHub Code Search",
+      prompt:
+        "Use [GitHub's Code Search syntax](https://docs.github.com/en/search-github/github-code-search/understanding-github-code-search-syntax), including code qualifiers, regular expressions, and boolean operations, to search for repositories.",
+      placeHolder: "org:github",
+    });
+    if (codeSearchQuery === undefined || codeSearchQuery === "") {
+      return;
+    }
+
+    void window.withProgress(
+      {
+        location: ProgressLocation.Notification,
+        title: "Searching for repositories... This might take a while",
+        cancellable: true,
+      },
+      async (progress, token) => {
+        progress.report({ increment: 10 });
+
+        const repositories = await getCodeSearchRepositories(
+          this.app.credentials,
+          `${codeSearchQuery} language:${codeSearchLanguage.language}`,
+          progress,
+          token,
+        );
+
+        token.onCancellationRequested(() => {
+          void showAndLogInformationMessage("Code search cancelled");
+          return;
+        });
+
+        progress.report({ increment: 10, message: "Processing results..." });
+
+        const truncatedRepositories =
+          await this.dbManager.addNewRemoteReposToList(repositories, listName);
+        this.reportAnyTruncatedRepos(truncatedRepositories, listName);
+      },
+    );
+  }
+
+  private reportAnyTruncatedRepos(
+    truncatedRepositories: string[],
+    listName: string,
+  ) {
+    if (truncatedRepositories.length > 0) {
+      void showAndLogErrorMessage(
+        `Some repositories were not added to '${listName}' because a list can only have 1000 entries. Excluded repositories: ${truncatedRepositories.join(
+          ", ",
+        )}`,
+      );
+    }
+  }
+
   private async onDidCollapseElement(
     event: TreeViewExpansionEvent<DbTreeViewItem>,
   ): Promise<void> {

extensions/ql-vscode/src/databases/ui/db-tree-view-item-action.ts

@@ -4,7 +4,8 @@ export type DbTreeViewItemAction =
   | "canBeSelected"
   | "canBeRemoved"
   | "canBeRenamed"
-  | "canBeOpenedOnGitHub";
+  | "canBeOpenedOnGitHub"
+  | "canImportCodeSearch";
 
 export function getDbItemActions(dbItem: DbItem): DbTreeViewItemAction[] {
   const actions: DbTreeViewItemAction[] = [];
@@ -21,7 +22,9 @@ export function getDbItemActions(dbItem: DbItem): DbTreeViewItemAction[] {
   if (canBeOpenedOnGitHub(dbItem)) {
     actions.push("canBeOpenedOnGitHub");
   }
-
+  if (canImportCodeSearch(dbItem)) {
+    actions.push("canImportCodeSearch");
+  }
   return actions;
 }
 
@@ -60,6 +63,10 @@ function canBeOpenedOnGitHub(dbItem: DbItem): boolean {
   return dbItemKindsThatCanBeOpenedOnGitHub.includes(dbItem.kind);
 }
 
+function canImportCodeSearch(dbItem: DbItem): boolean {
+  return DbItemKind.RemoteUserDefinedList === dbItem.kind;
+}
+
 export function getGitHubUrl(dbItem: DbItem): string | undefined {
   switch (dbItem.kind) {
     case DbItemKind.RemoteOwner:

extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts

@@ -7,6 +7,43 @@ import {
   VariantAnalysisSubmissionRequest,
 } from "./variant-analysis";
 import { Repository } from "./repository";
+import { Progress } from "vscode";
+import { CancellationToken } from "vscode-jsonrpc";
+
+export async function getCodeSearchRepositories(
+  credentials: Credentials,
+  query: string,
+  progress: Progress<{
+    message?: string | undefined;
+    increment?: number | undefined;
+  }>,
+  token: CancellationToken,
+): Promise<string[]> {
+  let nwos: string[] = [];
+  const octokit = await credentials.getOctokit();
+  for await (const response of octokit.paginate.iterator(
+    octokit.rest.search.repos,
+    {
+      q: query,
+      per_page: 100,
+    },
+  )) {
+    nwos.push(...response.data.map((item) => item.full_name));
+    // calculate progress bar: 80% of the progress bar is used for the code search
+    const totalNumberOfRequests = Math.ceil(response.data.total_count / 100);
+    // Since we have a maximum 10 of requests, we use a fixed increment whenever the totalNumberOfRequests is greater than 10
+    const increment =
+      totalNumberOfRequests < 10 ? 80 / totalNumberOfRequests : 8;
+    progress.report({ increment });
+
+    if (token.isCancellationRequested) {
+      nwos = [];
+      break;
+    }
+  }
+
+  return [...new Set(nwos)];
+}
 
 export async function submitVariantAnalysis(
   credentials: Credentials,