Split flows for running remote queries

extensions/ql-vscode/src/extension.ts

@@ -33,6 +33,7 @@ import {
   CliConfigListener,
   DistributionConfigListener,
   isCanary,
+  isVariantAnalysisLiveResultsEnabled,
   joinOrderWarningThreshold,
   MAX_QUERIES,
   QueryHistoryConfigListener,
@@ -489,13 +490,13 @@ async function activateWithInstalledDistribution(
   const variantAnalysisStorageDir = path.join(ctx.globalStorageUri.fsPath, 'variant-analyses');
   await fs.ensureDir(variantAnalysisStorageDir);
   const variantAnalysisResultsManager = new VariantAnalysisResultsManager(cliServer, logger);
-  const variantAnalysisManager = new VariantAnalysisManager(ctx, variantAnalysisStorageDir, variantAnalysisResultsManager);
+  const variantAnalysisManager = new VariantAnalysisManager(ctx, cliServer, variantAnalysisStorageDir, variantAnalysisResultsManager);
   ctx.subscriptions.push(variantAnalysisManager);
   ctx.subscriptions.push(variantAnalysisResultsManager);
   ctx.subscriptions.push(workspace.registerTextDocumentContentProvider('codeql-variant-analysis', createVariantAnalysisContentProvider(variantAnalysisManager)));
 
   void logger.log('Initializing remote queries manager.');
-  const rqm = new RemoteQueriesManager(ctx, cliServer, queryStorageDir, logger, variantAnalysisManager);
+  const rqm = new RemoteQueriesManager(ctx, cliServer, queryStorageDir, logger);
   ctx.subscriptions.push(rqm);
 
   void logger.log('Initializing query history.');
@@ -906,11 +907,20 @@ async function activateWithInstalledDistribution(
           step: 0,
           message: 'Getting credentials'
         });
-        await rqm.runRemoteQuery(
-          uri || window.activeTextEditor?.document.uri,
-          progress,
-          token
-        );
+
+        if (isVariantAnalysisLiveResultsEnabled()) {
+          await variantAnalysisManager.runVariantAnalysis(
+            uri || window.activeTextEditor?.document.uri,
+            progress,
+            token
+          );
+        } else {
+          await rqm.runRemoteQuery(
+            uri || window.activeTextEditor?.document.uri,
+            progress,
+            token
+          );
+        }
       } else {
         throw new Error('Variant analysis requires the CodeQL Canary version to run.');
       }

extensions/ql-vscode/src/remote-queries/remote-queries-api.ts

@@ -0,0 +1,87 @@
+import * as os from 'os';
+import { Credentials } from '../authentication';
+import { RepositorySelection } from './repository-selection';
+import { Repository } from './shared/repository';
+import { RemoteQueriesResponse } from './gh-api/remote-queries';
+import * as ghApiClient from './gh-api/gh-api-client';
+import { showAndLogErrorMessage, showAndLogInformationMessage } from '../helpers';
+import { getErrorMessage } from '../pure/helpers-pure';
+import { pluralize } from '../pure/word';
+
+export async function runRemoteQueriesApiRequest(
+  credentials: Credentials,
+  ref: string,
+  language: string,
+  repoSelection: RepositorySelection,
+  controllerRepo: Repository,
+  queryPackBase64: string,
+): Promise<void | RemoteQueriesResponse> {
+  try {
+    const response = await ghApiClient.submitRemoteQueries(credentials, {
+      ref,
+      language,
+      repositories: repoSelection.repositories,
+      repositoryLists: repoSelection.repositoryLists,
+      repositoryOwners: repoSelection.owners,
+      queryPack: queryPackBase64,
+      controllerRepoId: controllerRepo.id,
+    });
+    const { popupMessage, logMessage } = parseResponse(controllerRepo, response);
+    void showAndLogInformationMessage(popupMessage, { fullMessage: logMessage });
+    return response;
+  } catch (error: any) {
+    if (error.status === 404) {
+      void showAndLogErrorMessage(`Controller repository was not found. Please make sure it's a valid repo name.${eol}`);
+    } else {
+      void showAndLogErrorMessage(getErrorMessage(error));
+    }
+  }
+}
+
+const eol = os.EOL;
+const eol2 = os.EOL + os.EOL;
+
+// exported for testing only
+export function parseResponse(controllerRepo: Repository, response: RemoteQueriesResponse) {
+  const repositoriesQueried = response.repositories_queried;
+  const repositoryCount = repositoriesQueried.length;
+
+  const popupMessage = `Successfully scheduled runs on ${pluralize(repositoryCount, 'repository', 'repositories')}. [Click here to see the progress](https://github.com/${controllerRepo.fullName}/actions/runs/${response.workflow_run_id}).`
+    + (response.errors ? `${eol2}Some repositories could not be scheduled. See extension log for details.` : '');
+
+  let logMessage = `Successfully scheduled runs on ${pluralize(repositoryCount, 'repository', 'repositories')}. See https://github.com/${controllerRepo.fullName}/actions/runs/${response.workflow_run_id}.`;
+  logMessage += `${eol2}Repositories queried:${eol}${repositoriesQueried.join(', ')}`;
+  if (response.errors) {
+    const { invalid_repositories, repositories_without_database, private_repositories, cutoff_repositories, cutoff_repositories_count } = response.errors;
+    logMessage += `${eol2}Some repositories could not be scheduled.`;
+    if (invalid_repositories?.length) {
+      logMessage += `${eol2}${pluralize(invalid_repositories.length, 'repository', 'repositories')} invalid and could not be found:${eol}${invalid_repositories.join(', ')}`;
+    }
+    if (repositories_without_database?.length) {
+      logMessage += `${eol2}${pluralize(repositories_without_database.length, 'repository', 'repositories')} did not have a CodeQL database available:${eol}${repositories_without_database.join(', ')}`;
+      logMessage += `${eol}For each public repository that has not yet been added to the database service, we will try to create a database next time the store is updated.`;
+    }
+    if (private_repositories?.length) {
+      logMessage += `${eol2}${pluralize(private_repositories.length, 'repository', 'repositories')} not public:${eol}${private_repositories.join(', ')}`;
+      logMessage += `${eol}When using a public controller repository, only public repositories can be queried.`;
+    }
+    if (cutoff_repositories_count) {
+      logMessage += `${eol2}${pluralize(cutoff_repositories_count, 'repository', 'repositories')} over the limit for a single request`;
+      if (cutoff_repositories) {
+        logMessage += `:${eol}${cutoff_repositories.join(', ')}`;
+        if (cutoff_repositories_count !== cutoff_repositories.length) {
+          const moreRepositories = cutoff_repositories_count - cutoff_repositories.length;
+          logMessage += `${eol}...${eol}And another ${pluralize(moreRepositories, 'repository', 'repositories')}.`;
+        }
+      } else {
+        logMessage += '.';
+      }
+      logMessage += `${eol}Repositories were selected based on how recently they had been updated.`;
+    }
+  }
+
+  return {
+    popupMessage,
+    logMessage
+  };
+}

extensions/ql-vscode/src/remote-queries/remote-queries-manager.ts

@@ -1,4 +1,4 @@
-import { CancellationToken, commands, EventEmitter, ExtensionContext, Uri, env, window } from 'vscode';
+import { CancellationToken, commands, EventEmitter, ExtensionContext, Uri, env } from 'vscode';
 import { nanoid } from 'nanoid';
 import * as path from 'path';
 import * as fs from 'fs-extra';
@@ -9,9 +9,11 @@ import { CodeQLCliServer } from '../cli';
 import { ProgressCallback } from '../commandRunner';
 import { createTimestampFile, showAndLogErrorMessage, showAndLogInformationMessage, showInformationMessageWithAction } from '../helpers';
 import { Logger } from '../logging';
-import { runRemoteQuery } from './run-remote-query';
+import {
+  prepareRemoteQueryRun,
+} from './run-remote-query';
 import { RemoteQueriesView } from './remote-queries-view';
-import { RemoteQuery } from './remote-query';
+import { buildRemoteQueryEntity, RemoteQuery } from './remote-query';
 import { RemoteQueriesMonitor } from './remote-queries-monitor';
 import { getRemoteQueryIndex, getRepositoriesMetadata, RepositoriesMetadata } from './gh-api/gh-actions-api-client';
 import { RemoteQueryResultIndex } from './remote-query-result-index';
@@ -22,7 +24,7 @@ import { assertNever } from '../pure/helpers-pure';
 import { QueryStatus } from '../query-status';
 import { DisposableObject } from '../pure/disposable-object';
 import { AnalysisResults } from './shared/analysis-result';
-import { VariantAnalysisManager } from './variant-analysis-manager';
+import { runRemoteQueriesApiRequest } from './remote-queries-api';
 
 const autoDownloadMaxSize = 300 * 1024;
 const autoDownloadMaxCount = 100;
@@ -57,21 +59,18 @@ export class RemoteQueriesManager extends DisposableObject {
 
   private readonly remoteQueriesMonitor: RemoteQueriesMonitor;
   private readonly analysesResultsManager: AnalysesResultsManager;
-  private readonly variantAnalysisManager: VariantAnalysisManager;
   private readonly view: RemoteQueriesView;
 
   constructor(
     private readonly ctx: ExtensionContext,
     private readonly cliServer: CodeQLCliServer,
     private readonly storagePath: string,
     logger: Logger,
-    variantAnalysisManager: VariantAnalysisManager,
   ) {
     super();
     this.analysesResultsManager = new AnalysesResultsManager(ctx, cliServer, storagePath, logger);
     this.view = new RemoteQueriesView(ctx, logger, this.analysesResultsManager);
     this.remoteQueriesMonitor = new RemoteQueriesMonitor(ctx, logger);
-    this.variantAnalysisManager = variantAnalysisManager;
 
     this.remoteQueryAddedEventEmitter = this.push(new EventEmitter<NewQueryEvent>());
     this.remoteQueryRemovedEventEmitter = this.push(new EventEmitter<RemovedQueryEvent>());
@@ -122,18 +121,42 @@ export class RemoteQueriesManager extends DisposableObject {
   ): Promise<void> {
     const credentials = await Credentials.initialize(this.ctx);
 
-    const querySubmission = await runRemoteQuery(this.cliServer, credentials, uri || window.activeTextEditor?.document.uri, progress, token, this.variantAnalysisManager);
-
-    if (querySubmission?.query) {
-      const query = querySubmission.query;
-      const queryId = this.createQueryId();
-
-      await this.prepareStorageDirectory(queryId);
-      await this.storeJsonFile(queryId, 'query.json', query);
-
-      this.remoteQueryAddedEventEmitter.fire({ queryId, query });
-      void commands.executeCommand('codeQL.monitorRemoteQuery', queryId, query);
+    const {
+      actionBranch,
+      base64Pack,
+      repoSelection,
+      queryFile,
+      queryMetadata,
+      controllerRepo,
+      queryStartTime,
+      language,
+    } = await prepareRemoteQueryRun(this.cliServer, credentials, uri, progress, token);
+
+    const apiResponse = await runRemoteQueriesApiRequest(credentials, actionBranch, language, repoSelection, controllerRepo, base64Pack);
+
+    if (!apiResponse) {
+      return;
     }
+
+    const workflowRunId = apiResponse.workflow_run_id;
+    const repositoryCount = apiResponse.repositories_queried.length;
+    const query = await buildRemoteQueryEntity(
+      queryFile,
+      queryMetadata,
+      controllerRepo,
+      queryStartTime,
+      workflowRunId,
+      language,
+      repositoryCount
+    );
+
+    const queryId = this.createQueryId();
+
+    await this.prepareStorageDirectory(queryId);
+    await this.storeJsonFile(queryId, 'query.json', query);
+
+    this.remoteQueryAddedEventEmitter.fire({ queryId, query });
+    void commands.executeCommand('codeQL.monitorRemoteQuery', queryId, query);
   }
 
   public async monitorRemoteQuery(

extensions/ql-vscode/src/remote-queries/remote-query.ts

@@ -1,12 +1,44 @@
-import { Repository } from './repository';
+import * as fs from 'fs-extra';
+import { Repository as RemoteRepository } from './repository';
+import { QueryMetadata } from '../pure/interface-types';
+import { getQueryName } from './run-remote-query';
+import { Repository } from './shared/repository';
 
 export interface RemoteQuery {
   queryName: string;
   queryFilePath: string;
   queryText: string;
   language: string;
-  controllerRepository: Repository;
+  controllerRepository: RemoteRepository;
   executionStartTime: number; // Use number here since it needs to be serialized and desserialized.
   actionsWorkflowRunId: number;
   repositoryCount: number;
 }
+
+export async function buildRemoteQueryEntity(
+  queryFilePath: string,
+  queryMetadata: QueryMetadata | undefined,
+  controllerRepo: Repository,
+  queryStartTime: number,
+  workflowRunId: number,
+  language: string,
+  repositoryCount: number
+): Promise<RemoteQuery> {
+  const queryName = getQueryName(queryMetadata, queryFilePath);
+  const queryText = await fs.readFile(queryFilePath, 'utf8');
+  const [owner, name] = controllerRepo.fullName.split('/');
+
+  return {
+    queryName,
+    queryFilePath,
+    queryText,
+    language,
+    controllerRepository: {
+      owner,
+      name,
+    },
+    executionStartTime: queryStartTime,
+    actionsWorkflowRunId: workflowRunId,
+    repositoryCount,
+  };
+}