From f903a5ddb6f1e32320b5e79e65258b7657dac4eb Mon Sep 17 00:00:00 2001 From: Philip Ginsbach Date: Mon, 10 Nov 2025 16:03:29 +0000 Subject: [PATCH 1/7] add commands for warming the overlay-base cache --- extensions/ql-vscode/package.json | 46 ++++++++ extensions/ql-vscode/src/common/commands.ts | 6 ++ .../src/common/logging/vscode/loggers.ts | 4 + .../local-databases/database-manager.ts | 29 +++++ extensions/ql-vscode/src/extension.ts | 42 +++++++- .../ast-viewer/ast-cfg-commands.ts | 1 + .../src/local-queries/local-queries.ts | 101 ++++++++++++++++-- .../src/local-queries/local-query-run.ts | 10 +- .../src/query-server/query-server-client.ts | 33 ++++-- .../local-queries/local-databases.test.ts | 43 ++++++++ 10 files changed, 292 insertions(+), 23 deletions(-) diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 81dee5f6343..73a75cba972 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -515,6 +515,14 @@ "command": "codeQL.runQueryContextEditor", "title": "CodeQL: Run Query on Selected Database" }, + { + "command": "codeQL.runWarmOverlayBaseCacheForQuery", + "title": "CodeQL: Warm Overlay-Base Cache for Query" + }, + { + "command": "codeQL.runWarmOverlayBaseCacheForQueryContextEditor", + "title": "CodeQL: Warm Overlay-Base Cache for Query" + }, { "command": "codeQL.debugQuery", "title": "CodeQL: Debug Query" @@ -571,10 +579,18 @@ "command": "codeQL.runQueries", "title": "CodeQL: Run Queries in Selected Files" }, + { + "command": "codeQL.runWarmOverlayBaseCacheForQueries", + "title": "CodeQL: Warm Overlay-Base Cache for Queries in Selected Files" + }, { "command": "codeQL.runQuerySuite", "title": "CodeQL: Run Selected Query Suite" }, + { + "command": "codeQL.runWarmOverlayBaseCacheForQuerySuite", + "title": "CodeQL: Warm Overlay-Base Cache for Query Suite" + }, { "command": "codeQL.quickEval", "title": "CodeQL: Quick Evaluation" @@ -1378,11 +1394,21 @@ "group": "9_qlCommands", "when": "resourceScheme != codeql-zip-archive" }, + { + "command": "codeQL.runWarmOverlayBaseCacheForQueries", + "group": "9_qlCommands", + "when": "resourceScheme != codeql-zip-archive" + }, { "command": "codeQL.runQuerySuite", "group": "9_qlCommands", "when": "resourceScheme != codeql-zip-archive && resourceExtname == .qls && !explorerResourceIsFolder && !listMultiSelection && config.codeQL.canary" }, + { + "command": "codeQL.runWarmOverlayBaseCacheForQuerySuite", + "group": "9_qlCommands", + "when": "resourceScheme != codeql-zip-archive && resourceExtname == .qls && !explorerResourceIsFolder && !listMultiSelection && config.codeQL.canary" + }, { "command": "codeQL.runVariantAnalysisContextExplorer", "group": "9_qlCommands", @@ -1408,6 +1434,10 @@ "command": "codeQL.runQuery", "when": "resourceLangId == ql && resourceExtname == .ql" }, + { + "command": "codeQL.runWarmOverlayBaseCacheForQuery", + "when": "resourceLangId == ql && resourceExtname == .ql" + }, { "command": "codeQLQueries.runLocalQueryFromQueriesPanel", "when": "false" @@ -1428,6 +1458,10 @@ "command": "codeQL.runQueryContextEditor", "when": "false" }, + { + "command": "codeQL.runWarmOverlayBaseCacheForQueryContextEditor", + "when": "false" + }, { "command": "codeQL.debugQuery", "when": "config.codeQL.canary && editorLangId == ql && resourceExtname == .ql && !inDebugMode" @@ -1480,10 +1514,18 @@ "command": "codeQL.runQueries", "when": "false" }, + { + "command": "codeQL.runWarmOverlayBaseCacheForQueries", + "when": "false" + }, { "command": "codeQL.runQuerySuite", "when": "false" }, + { + "command": "codeQL.runWarmOverlayBaseCacheForQuerySuite", + "when": "false" + }, { "command": "codeQL.quickEval", "when": "editorLangId == ql" @@ -1832,6 +1874,10 @@ "command": "codeQL.runQueryContextEditor", "when": "editorLangId == ql && resourceExtname == .ql && !inDebugMode" }, + { + "command": "codeQL.runWarmOverlayBaseCacheForQueryContextEditor", + "when": "editorLangId == ql && resourceExtname == .ql && !inDebugMode" + }, { "command": "codeQL.runQueryOnMultipleDatabasesContextEditor", "when": "editorLangId == ql && resourceExtname == .ql" diff --git a/extensions/ql-vscode/src/common/commands.ts b/extensions/ql-vscode/src/common/commands.ts index 64585a8c9e8..f4126194aa2 100644 --- a/extensions/ql-vscode/src/common/commands.ts +++ b/extensions/ql-vscode/src/common/commands.ts @@ -126,7 +126,11 @@ export type QueryEditorCommands = { // Commands used for running local queries export type LocalQueryCommands = { "codeQL.runQuery": (uri?: Uri) => Promise; + "codeQL.runWarmOverlayBaseCacheForQuery": (uri?: Uri) => Promise; "codeQL.runQueryContextEditor": (uri?: Uri) => Promise; + "codeQL.runWarmOverlayBaseCacheForQueryContextEditor": ( + uri?: Uri, + ) => Promise; "codeQL.runQueryOnMultipleDatabases": (uri?: Uri) => Promise; "codeQL.runQueryOnMultipleDatabasesContextEditor": ( uri?: Uri, @@ -138,7 +142,9 @@ export type LocalQueryCommands = { "codeQLQueries.createQuery": () => Promise; "codeQL.runLocalQueryFromFileTab": (uri: Uri) => Promise; "codeQL.runQueries": ExplorerSelectionCommandFunction; + "codeQL.runWarmOverlayBaseCacheForQueries": ExplorerSelectionCommandFunction; "codeQL.runQuerySuite": ExplorerSelectionCommandFunction; + "codeQL.runWarmOverlayBaseCacheForQuerySuite": ExplorerSelectionCommandFunction; "codeQL.quickEval": (uri: Uri) => Promise; "codeQL.quickEvalCount": (uri: Uri) => Promise; "codeQL.quickEvalContextEditor": (uri: Uri) => Promise; diff --git a/extensions/ql-vscode/src/common/logging/vscode/loggers.ts b/extensions/ql-vscode/src/common/logging/vscode/loggers.ts index 8d4401dfec1..9cb953de6de 100644 --- a/extensions/ql-vscode/src/common/logging/vscode/loggers.ts +++ b/extensions/ql-vscode/src/common/logging/vscode/loggers.ts @@ -10,6 +10,10 @@ export const extLogger = new OutputChannelLogger("CodeQL Extension Log"); // Logger for messages from the query server. export const queryServerLogger = new OutputChannelLogger("CodeQL Query Server"); +// Logger for messages from the query server for warming overlay-base cache. +export const queryServerForWarmingOverlayBaseCacheLogger = + new OutputChannelLogger("CodeQL Query Server for warming overlay-base cache"); + // Logger for messages from the language server. export const languageServerLogger = new OutputChannelLogger( "CodeQL Language Server", diff --git a/extensions/ql-vscode/src/databases/local-databases/database-manager.ts b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts index 5685f5f997a..516e9fd6fff 100644 --- a/extensions/ql-vscode/src/databases/local-databases/database-manager.ts +++ b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts @@ -118,6 +118,7 @@ export class DatabaseManager extends DisposableObject { private readonly ctx: ExtensionContext, private readonly app: App, private readonly qs: QueryRunner, + private readonly qsForWarmingOverlayBaseCache: QueryRunner, private readonly cli: CodeQLCliServer, private readonly languageContext: LanguageContextStore, public logger: Logger, @@ -739,6 +740,34 @@ export class DatabaseManager extends DisposableObject { } } + public async withDatabaseInQsForWarmingOverlayBaseCache( + whatToDo: () => Promise, + ) { + try { + if (this._currentDatabaseItem) { + const dbItem = this._currentDatabaseItem; + await this.qs.deregisterDatabase(dbItem); + await this.qsForWarmingOverlayBaseCache.registerDatabase(dbItem); + } + await whatToDo(); + if (this._currentDatabaseItem) { + const dbItem = this._currentDatabaseItem; + await this.qsForWarmingOverlayBaseCache.deregisterDatabase(dbItem); + await this.qs.registerDatabase(dbItem); + } + } catch (e) { + const message = getErrorMessage(e); + if (message === "Connection is disposed.") { + // This is expected if the query server is not running. + void extLogger.log( + `Could not use database for warming overlay-base cache because query server is not running.`, + ); + return; + } + throw e; + } + } + private async deregisterDatabase(dbItem: DatabaseItem) { try { await this.qs.deregisterDatabase(dbItem); diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts index 5790a0b5dfb..a3f8088b26a 100644 --- a/extensions/ql-vscode/src/extension.ts +++ b/extensions/ql-vscode/src/extension.ts @@ -86,6 +86,7 @@ import { extLogger, languageServerLogger, queryServerLogger, + queryServerForWarmingOverlayBaseCacheLogger, } from "./common/logging/vscode"; import { QueryHistoryManager } from "./query-history/query-history-manager"; import type { CompletedLocalQueryInfo } from "./query-results"; @@ -172,6 +173,7 @@ function getCommands( app: App, cliServer: CodeQLCliServer, queryRunner: QueryRunner, + queryRunnerForWarmingOverlayBaseCache: QueryRunner, languageClient: LanguageClient, ): BaseCommands { const getCliVersion = async () => { @@ -189,6 +191,7 @@ function getCommands( cliServer.restartCliServer(); await Promise.all([ queryRunner.restartQueryServer(progress), + queryRunnerForWarmingOverlayBaseCache.restartQueryServer(progress), async () => { if (languageClient.isRunning()) { await languageClient.restart(); @@ -201,6 +204,10 @@ function getCommands( queryServerLogger, "CodeQL Query Server restarted.", ); + void showAndLogErrorMessage( + queryServerForWarmingOverlayBaseCacheLogger, + "CodeQL Query Server for warming overlay-base cache restarted.", + ); }, { title: "Restarting Query Server", @@ -281,6 +288,7 @@ export interface CodeQLExtensionInterface { readonly ctx: ExtensionContext; readonly cliServer: CodeQLCliServer; readonly qs: QueryRunner; + readonly qsForWarmingOverlayBaseCache: QueryRunner; readonly distributionManager: DistributionManager; readonly databaseManager: DatabaseManager; readonly databaseUI: DatabaseUI; @@ -795,6 +803,16 @@ async function activateWithInstalledDistribution( qlConfigurationListener, cliServer, ctx, + false, + ); + + void extLogger.log("Initializing base cache warming query server client."); + const qsForWarmingOverlayBaseCache = await createQueryServer( + app, + qlConfigurationListener, + cliServer, + ctx, + true, ); for (const glob of CLEAR_PACK_CACHE_ON_EDIT_GLOBS) { @@ -822,6 +840,7 @@ async function activateWithInstalledDistribution( ctx, app, qs, + qsForWarmingOverlayBaseCache, cliServer, languageContext, extLogger, @@ -998,6 +1017,7 @@ async function activateWithInstalledDistribution( const localQueries = new LocalQueries( app, qs, + qsForWarmingOverlayBaseCache, qhm, dbm, databaseFetcher, @@ -1062,7 +1082,13 @@ async function activateWithInstalledDistribution( void extLogger.log("Registering top-level command palette commands."); const allCommands: AllExtensionCommands = { - ...getCommands(app, cliServer, qs, languageClient), + ...getCommands( + app, + cliServer, + qs, + qsForWarmingOverlayBaseCache, + languageClient, + ), ...getQueryEditorCommands({ commandManager: app.commands, queryRunner: qs, @@ -1165,6 +1191,7 @@ async function activateWithInstalledDistribution( cliServer, localQueries, qs, + qsForWarmingOverlayBaseCache, distributionManager, databaseManager: dbm, databaseUI, @@ -1278,9 +1305,12 @@ async function createQueryServer( qlConfigurationListener: QueryServerConfigListener, cliServer: CodeQLCliServer, ctx: ExtensionContext, + warmOverlayBaseCache: boolean, ): Promise { const qsOpts = { - logger: queryServerLogger, + logger: warmOverlayBaseCache + ? queryServerForWarmingOverlayBaseCacheLogger + : queryServerLogger, contextStoragePath: getContextStoragePath(ctx), }; const progressCallback = ( @@ -1290,7 +1320,12 @@ async function createQueryServer( ) => Thenable, ) => Window.withProgress( - { title: "CodeQL query server", location: ProgressLocation.Window }, + { + title: warmOverlayBaseCache + ? "CodeQL query server for warming overlay-base cache" + : "CodeQL query server", + location: ProgressLocation.Window, + }, task, ); @@ -1300,6 +1335,7 @@ async function createQueryServer( cliServer, qsOpts, progressCallback, + warmOverlayBaseCache, ); ctx.subscriptions.push(qs); await qs.startQueryServer(); diff --git a/extensions/ql-vscode/src/language-support/ast-viewer/ast-cfg-commands.ts b/extensions/ql-vscode/src/language-support/ast-viewer/ast-cfg-commands.ts index 8a934505448..c6cf025832d 100644 --- a/extensions/ql-vscode/src/language-support/ast-viewer/ast-cfg-commands.ts +++ b/extensions/ql-vscode/src/language-support/ast-viewer/ast-cfg-commands.ts @@ -59,6 +59,7 @@ export function getAstCfgCommands({ progress, token, undefined, + false, undefined, res[1], ); diff --git a/extensions/ql-vscode/src/local-queries/local-queries.ts b/extensions/ql-vscode/src/local-queries/local-queries.ts index 4444ade8293..def3572a3f3 100644 --- a/extensions/ql-vscode/src/local-queries/local-queries.ts +++ b/extensions/ql-vscode/src/local-queries/local-queries.ts @@ -74,6 +74,7 @@ export class LocalQueries extends DisposableObject { public constructor( private readonly app: ExtensionApp, private readonly queryRunner: QueryRunner, + private readonly queryRunnerForWarmingOverlayBaseCache: QueryRunner, private readonly queryHistoryManager: QueryHistoryManager, private readonly databaseManager: DatabaseManager, private readonly databaseFetcher: DatabaseFetcher, @@ -95,7 +96,11 @@ export class LocalQueries extends DisposableObject { public getCommands(): LocalQueryCommands { return { "codeQL.runQuery": this.runQuery.bind(this), + "codeQL.runWarmOverlayBaseCacheForQuery": + this.runWarmOverlayBaseCacheForQuery.bind(this), "codeQL.runQueryContextEditor": this.runQuery.bind(this), + "codeQL.runWarmOverlayBaseCacheForQueryContextEditor": + this.runWarmOverlayBaseCacheForQuery.bind(this), "codeQL.runQueryOnMultipleDatabases": this.runQueryOnMultipleDatabases.bind(this), "codeQL.runQueryOnMultipleDatabasesContextEditor": @@ -113,7 +118,12 @@ export class LocalQueries extends DisposableObject { "codeQL.runQueries": createMultiSelectionCommand( this.runQueries.bind(this), ), + "codeQL.runWarmOverlayBaseCacheForQueries": createMultiSelectionCommand( + this.runWarmOverlayBaseCacheForQueries.bind(this), + ), "codeQL.runQuerySuite": this.runQuerySuite.bind(this), + "codeQL.runWarmOverlayBaseCacheForQuerySuite": + this.runWarmOverlayBaseCacheForQuerySuite.bind(this), "codeQL.quickEval": this.quickEval.bind(this), "codeQL.quickEvalCount": this.quickEvalCount.bind(this), "codeQL.quickEvalContextEditor": this.quickEval.bind(this), @@ -152,6 +162,20 @@ export class LocalQueries extends DisposableObject { } private async runQuery(uri: Uri | undefined): Promise { + await this.runQueryInternal(uri, false); + } + private async runWarmOverlayBaseCacheForQuery( + uri: Uri | undefined, + ): Promise { + await this.databaseManager.withDatabaseInQsForWarmingOverlayBaseCache(() => + this.runQueryInternal(uri, true), + ); + } + + private async runQueryInternal( + uri: Uri | undefined, + warmOverlayBaseCache: boolean, + ): Promise { await withProgress( async (progress, token) => { await this.compileAndRunQuery( @@ -160,10 +184,13 @@ export class LocalQueries extends DisposableObject { progress, token, undefined, + warmOverlayBaseCache, ); }, { - title: "Running query", + title: warmOverlayBaseCache + ? "Warm overlay-base cache for query" + : "Running query", cancellable: true, }, ); @@ -183,6 +210,21 @@ export class LocalQueries extends DisposableObject { } private async runQueries(fileURIs: Uri[]): Promise { + await this.runQueriesInternal(fileURIs, false); + } + + private async runWarmOverlayBaseCacheForQueries( + fileURIs: Uri[], + ): Promise { + await this.databaseManager.withDatabaseInQsForWarmingOverlayBaseCache(() => + this.runQueriesInternal(fileURIs, true), + ); + } + + private async runQueriesInternal( + fileURIs: Uri[], + warmOverlayBaseCache: boolean, + ): Promise { await withProgress( async (progress, token) => { const maxQueryCount = MAX_QUERIES.getValue(); @@ -235,18 +277,36 @@ export class LocalQueries extends DisposableObject { wrappedProgress, token, undefined, + warmOverlayBaseCache, ).then(() => queriesRemaining--), ), ); }, { - title: "Running queries", + title: warmOverlayBaseCache + ? "Warm overlay-base cache for queries" + : "Running queries", cancellable: true, }, ); } private async runQuerySuite(fileUri: Uri): Promise { + await this.runQuerySuiteInternal(fileUri, false); + } + + private async runWarmOverlayBaseCacheForQuerySuite( + fileUri: Uri, + ): Promise { + await this.databaseManager.withDatabaseInQsForWarmingOverlayBaseCache(() => + this.runQuerySuiteInternal(fileUri, true), + ); + } + + private async runQuerySuiteInternal( + fileUri: Uri, + warmOverlayBaseCache: boolean, + ): Promise { await withProgress( async (progress, token) => { const suitePath = validateQuerySuiteUri(fileUri); @@ -280,7 +340,10 @@ export class LocalQueries extends DisposableObject { quickEvalCountOnly: false, }); }); - const coreQueryRun = this.queryRunner.createQueryRun( + const queryRunner = warmOverlayBaseCache + ? this.queryRunnerForWarmingOverlayBaseCache + : this.queryRunner; + const coreQueryRun = queryRunner.createQueryRun( databaseItem.databaseUri.fsPath, queryTargets, true, @@ -301,6 +364,7 @@ export class LocalQueries extends DisposableObject { databaseItem, coreQueryRun.outputDir, source, + warmOverlayBaseCache, ); try { @@ -310,7 +374,11 @@ export class LocalQueries extends DisposableObject { localQueryRun.logger, ); - await localQueryRun.complete(results, progress); + await localQueryRun.complete( + results, + progress, + warmOverlayBaseCache, + ); return results; } catch (e) { @@ -328,7 +396,9 @@ export class LocalQueries extends DisposableObject { } }, { - title: "Running query suite", + title: warmOverlayBaseCache + ? "Warm overlay-base cache for query suite" + : "Running query suite", cancellable: true, }, ); @@ -379,6 +449,7 @@ export class LocalQueries extends DisposableObject { progress, token, undefined, + false, range, ), { @@ -447,10 +518,15 @@ export class LocalQueries extends DisposableObject { dbItem: DatabaseItem, outputDir: QueryOutputDir, tokenSource: CancellationTokenSource, + warmOverlayBaseCache: boolean = false, ): Promise { await createTimestampFile(outputDir.querySaveDir); - if (this.queryRunner.customLogDirectory) { + const queryRunner = warmOverlayBaseCache + ? this.queryRunnerForWarmingOverlayBaseCache + : this.queryRunner; + + if (queryRunner.customLogDirectory) { void showAndLogWarningMessage( this.app.logger, `Custom log directories are no longer supported. The "codeQL.runningQueries.customLogDirectory" setting is deprecated. Unset the setting to stop seeing this message. Query logs saved to ${outputDir.logPath}`, @@ -471,7 +547,7 @@ export class LocalQueries extends DisposableObject { const queryInfo = new LocalQueryInfo(initialInfo, tokenSource); this.queryHistoryManager.addQuery(queryInfo); - const logger = new TeeLogger(this.queryRunner.logger, outputDir.logPath); + const logger = new TeeLogger(queryRunner.logger, outputDir.logPath); return new LocalQueryRun( outputDir, this, @@ -489,6 +565,7 @@ export class LocalQueries extends DisposableObject { progress: ProgressCallback, token: CancellationToken, databaseItem: DatabaseItem | undefined, + warmOverlayBaseCache: boolean = false, range?: Range, templates?: Record, ): Promise { @@ -500,6 +577,7 @@ export class LocalQueries extends DisposableObject { databaseItem, range, templates, + warmOverlayBaseCache, ); } @@ -512,6 +590,7 @@ export class LocalQueries extends DisposableObject { databaseItem: DatabaseItem | undefined, range?: Range, templates?: Record, + warmOverlayBaseCache: boolean = false, ): Promise { await saveBeforeStart(); @@ -545,7 +624,10 @@ export class LocalQueries extends DisposableObject { const additionalPacks = getOnDiskWorkspaceFolders(); const extensionPacks = await this.getDefaultExtensionPacks(additionalPacks); - const coreQueryRun = this.queryRunner.createQueryRun( + const queryRunner = warmOverlayBaseCache + ? this.queryRunnerForWarmingOverlayBaseCache + : this.queryRunner; + const coreQueryRun = queryRunner.createQueryRun( databaseItem.databaseUri.fsPath, [ { @@ -574,6 +656,7 @@ export class LocalQueries extends DisposableObject { databaseItem, coreQueryRun.outputDir, source, + warmOverlayBaseCache, ); try { @@ -583,7 +666,7 @@ export class LocalQueries extends DisposableObject { localQueryRun.logger, ); - await localQueryRun.complete(results, progress); + await localQueryRun.complete(results, progress, warmOverlayBaseCache); return results; } catch (e) { diff --git a/extensions/ql-vscode/src/local-queries/local-query-run.ts b/extensions/ql-vscode/src/local-queries/local-query-run.ts index c2d6dec5972..4cfde130276 100644 --- a/extensions/ql-vscode/src/local-queries/local-query-run.ts +++ b/extensions/ql-vscode/src/local-queries/local-query-run.ts @@ -84,6 +84,7 @@ export class LocalQueryRun { public async complete( results: CoreQueryResults, progress: ProgressCallback, + warmOverlayBaseCache: boolean = false, ): Promise { const evalLogPaths = await this.summarizeEvalLog( Array.from(results.results.values()).every( @@ -104,10 +105,11 @@ export class LocalQueryRun { queriesWithResults, ); progress(progressUpdate(3, 4, "Showing results")); - await this.localQueries.showResultsForCompletedQuery( - this.queryInfo as CompletedLocalQueryInfo, - WebviewReveal.Forced, - ); + if (!warmOverlayBaseCache) + await this.localQueries.showResultsForCompletedQuery( + this.queryInfo as CompletedLocalQueryInfo, + WebviewReveal.Forced, + ); // Note we must update the query history view after showing results as the // display and sorting might depend on the number of results progress(progressUpdate(4, 4, "Updating query history")); diff --git a/extensions/ql-vscode/src/query-server/query-server-client.ts b/extensions/ql-vscode/src/query-server/query-server-client.ts index e62637ff3ad..3eb18e1194c 100644 --- a/extensions/ql-vscode/src/query-server/query-server-client.ts +++ b/extensions/ql-vscode/src/query-server/query-server-client.ts @@ -72,6 +72,7 @@ export class QueryServerClient extends DisposableObject { readonly cliServer: CodeQLCliServer, readonly opts: ServerOpts, withProgressReporting: WithProgressReporting, + readonly warmOverlayBaseCache: boolean = false, ) { super(); // Since no query is active when we initialize, just point the "active query logger" to the @@ -214,9 +215,21 @@ export class QueryServerClient extends DisposableObject { ); } + if (this.warmOverlayBaseCache) { + args.push( + "--no-evaluate-as-overlay", + "--cache-at-frontier", + "--warm-cache-only", + ); + } + + const queryServerSuffix = this.warmOverlayBaseCache + ? " for warming overlay-base cache" + : ""; + const child = spawnServer( this.config.codeQlPath, - "CodeQL query server", + `CodeQL query server${queryServerSuffix}`, ["execute", "query-server2"], args, this.logger, @@ -227,7 +240,9 @@ export class QueryServerClient extends DisposableObject { undefined, // no listener for stdout progressReporter, ); - progressReporter.report({ message: "Connecting to CodeQL query server" }); + progressReporter.report({ + message: `Connecting to CodeQL query server${queryServerSuffix}`, + }); const connection = createMessageConnection(child.stdout, child.stdin); connection.onNotification(progress, (res) => { const callback = this.progressCallbacks[res.id]; @@ -238,13 +253,15 @@ export class QueryServerClient extends DisposableObject { this.serverProcess = new ServerProcess( child, connection, - "Query Server 2", + `Query Server 2${queryServerSuffix}`, this.logger, ); // Ensure the server process is disposed together with this client. this.track(this.serverProcess); connection.listen(); - progressReporter.report({ message: "Connected to CodeQL query server v2" }); + progressReporter.report({ + message: `Connected to CodeQL query server${queryServerSuffix} v2`, + }); this.nextCallback = 0; this.nextProgress = 0; this.progressCallbacks = {}; @@ -254,7 +271,9 @@ export class QueryServerClient extends DisposableObject { let wasExitOrErrorHandled = false; child.on("error", (err: Error) => { if (!wasExitOrErrorHandled) { - void this.logger.log(`Query server terminated with error: ${err}.`); + void this.logger.log( + `Query server${queryServerSuffix} terminated with error: ${err}.`, + ); this.restartQueryServerOnFailure(); wasExitOrErrorHandled = true; } @@ -263,12 +282,12 @@ export class QueryServerClient extends DisposableObject { if (!wasExitOrErrorHandled) { if (code !== null) { void this.logger.log( - `Query server terminated with exit code: ${code}.`, + `Query server${queryServerSuffix} terminated with exit code: ${code}.`, ); } if (signal !== null) { void this.logger.log( - `Query server terminated due to receipt of signal: ${signal}.`, + `Query server${queryServerSuffix} terminated due to receipt of signal: ${signal}.`, ); } this.restartQueryServerOnFailure(); diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts index b72be548844..516c2784861 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts @@ -44,6 +44,8 @@ describe("local databases", () => { let updateSpy: jest.Mock, []>; let registerSpy: jest.Mock, []>; let deregisterSpy: jest.Mock, []>; + let registerSpy2: jest.Mock, []>; + let deregisterSpy2: jest.Mock, []>; let resolveDatabaseSpy: jest.Mock, []>; let packAddSpy: jest.Mock; let logSpy: jest.Mock; @@ -63,6 +65,8 @@ describe("local databases", () => { updateSpy = jest.fn(() => Promise.resolve(undefined)); registerSpy = jest.fn(() => Promise.resolve(undefined)); deregisterSpy = jest.fn(() => Promise.resolve(undefined)); + registerSpy2 = jest.fn(() => Promise.resolve(undefined)); + deregisterSpy2 = jest.fn(() => Promise.resolve(undefined)); resolveDatabaseSpy = jest.fn(() => Promise.resolve({} as DbInfo)); packAddSpy = jest.fn(); logSpy = jest.fn(() => { @@ -105,6 +109,16 @@ describe("local databases", () => { /**/ }, }), + mockedObject({ + registerDatabase: registerSpy2, + deregisterDatabase: deregisterSpy2, + onStart: () => { + /**/ + }, + onQueryRunStarting: () => { + /**/ + }, + }), mockedObject({ resolveDatabase: resolveDatabaseSpy, packAdd: packAddSpy, @@ -324,6 +338,35 @@ describe("local databases", () => { // Should have deregistered this database expect(deregisterSpy).toHaveBeenCalledWith(mockDbItem); }); + + it("Should move database to secondary query server and back", async () => { + // similar test as above, but also check the call to sendRequestSpy to make sure they send the + // registration messages. + const mockDbItem = createMockDB(dir); + + await (databaseManager as any).addDatabaseItem(mockDbItem); + + await databaseManager.setCurrentDatabaseItem(mockDbItem, true); + // Should have registered this database + expect(registerSpy).toHaveBeenCalledWith(mockDbItem); + + await databaseManager.withDatabaseInQsForWarmingOverlayBaseCache( + async () => { + // Should have moved database registration + expect(deregisterSpy).toHaveBeenCalledWith(mockDbItem); + expect(registerSpy2).toHaveBeenCalledWith(mockDbItem); + }, + ); + + // Should have moved database registration back + expect(deregisterSpy2).toHaveBeenCalledWith(mockDbItem); + expect(registerSpy).toHaveBeenCalledWith(mockDbItem); + + await databaseManager.removeDatabaseItem(mockDbItem); + + // Should have deregistered this database + expect(deregisterSpy).toHaveBeenCalledWith(mockDbItem); + }); }); describe("resolveSourceFile", () => { From 236fd4b05f6ff64d048c5d9db7025afbe9f046d6 Mon Sep 17 00:00:00 2001 From: Philip Ginsbach Date: Tue, 18 Nov 2025 13:55:31 +0000 Subject: [PATCH 2/7] add changelog entry for overlay-base cache warming --- extensions/ql-vscode/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extensions/ql-vscode/CHANGELOG.md b/extensions/ql-vscode/CHANGELOG.md index f1ffd4f9f5c..2db288ac665 100644 --- a/extensions/ql-vscode/CHANGELOG.md +++ b/extensions/ql-vscode/CHANGELOG.md @@ -2,6 +2,8 @@ ## [UNRELEASED] +- Add new commands "CodeQL: Warm Overlay-Base Cache for Query", "CodeQL: Warm Overlay-Base Cache for Queries in Selected Files", and "CodeQL: Warm Overlay-Base Cache for Query Suite". These commands populate the overlay-base cache of an overlay database to prepare for incremental evaluation. They are primarily intended for overlay-specific performance debugging. [#4195](https://github.com/github/vscode-codeql/pull/4195) + ## 1.17.6 - 24 October 2025 - Remove support for CodeQL CLI versions older than 2.20.7. [#4159](https://github.com/github/vscode-codeql/pull/4159) From bc2cbe42043641b76c876e13e2dd525c595bd8c7 Mon Sep 17 00:00:00 2001 From: Philip Ginsbach-Chen Date: Wed, 19 Nov 2025 14:37:11 +0000 Subject: [PATCH 3/7] Update extensions/ql-vscode/package.json Co-authored-by: Nick Rolfe --- extensions/ql-vscode/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 73a75cba972..dc713c8d3a1 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -1397,7 +1397,7 @@ { "command": "codeQL.runWarmOverlayBaseCacheForQueries", "group": "9_qlCommands", - "when": "resourceScheme != codeql-zip-archive" + "when": "resourceScheme != codeql-zip-archive && config.codeQL.canary" }, { "command": "codeQL.runQuerySuite", From 17595540acb63951f2ea501d6665b21c8511ede8 Mon Sep 17 00:00:00 2001 From: Philip Ginsbach-Chen Date: Wed, 19 Nov 2025 14:37:20 +0000 Subject: [PATCH 4/7] Update extensions/ql-vscode/package.json Co-authored-by: Nick Rolfe --- extensions/ql-vscode/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index dc713c8d3a1..5af8185c6bd 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -1436,7 +1436,7 @@ }, { "command": "codeQL.runWarmOverlayBaseCacheForQuery", - "when": "resourceLangId == ql && resourceExtname == .ql" + "when": "resourceLangId == ql && resourceExtname == .ql && config.codeQL.canary" }, { "command": "codeQLQueries.runLocalQueryFromQueriesPanel", From fd8a80b88308b945cb6981884f999f704e9c5049 Mon Sep 17 00:00:00 2001 From: Philip Ginsbach-Chen Date: Wed, 19 Nov 2025 14:37:30 +0000 Subject: [PATCH 5/7] Update extensions/ql-vscode/package.json Co-authored-by: Nick Rolfe --- extensions/ql-vscode/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 5af8185c6bd..469a408667c 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -1876,7 +1876,7 @@ }, { "command": "codeQL.runWarmOverlayBaseCacheForQueryContextEditor", - "when": "editorLangId == ql && resourceExtname == .ql && !inDebugMode" + "when": "editorLangId == ql && resourceExtname == .ql && !inDebugMode && config.codeQL.canary" }, { "command": "codeQL.runQueryOnMultipleDatabasesContextEditor", From 038deffe10a353af04427cdfb4decd76c7341de7 Mon Sep 17 00:00:00 2001 From: Philip Ginsbach Date: Wed, 19 Nov 2025 14:38:27 +0000 Subject: [PATCH 6/7] remove changelog entry for changes behind canary flag --- extensions/ql-vscode/CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/extensions/ql-vscode/CHANGELOG.md b/extensions/ql-vscode/CHANGELOG.md index 2db288ac665..f1ffd4f9f5c 100644 --- a/extensions/ql-vscode/CHANGELOG.md +++ b/extensions/ql-vscode/CHANGELOG.md @@ -2,8 +2,6 @@ ## [UNRELEASED] -- Add new commands "CodeQL: Warm Overlay-Base Cache for Query", "CodeQL: Warm Overlay-Base Cache for Queries in Selected Files", and "CodeQL: Warm Overlay-Base Cache for Query Suite". These commands populate the overlay-base cache of an overlay database to prepare for incremental evaluation. They are primarily intended for overlay-specific performance debugging. [#4195](https://github.com/github/vscode-codeql/pull/4195) - ## 1.17.6 - 24 October 2025 - Remove support for CodeQL CLI versions older than 2.20.7. [#4159](https://github.com/github/vscode-codeql/pull/4159) From 56f8652872d32c6a92504f4308d65d943eb12109 Mon Sep 17 00:00:00 2001 From: Philip Ginsbach Date: Wed, 19 Nov 2025 15:40:06 +0000 Subject: [PATCH 7/7] construct qsForWarmingOverlayBaseCache lazily --- .../src/common/logging/vscode/loggers.ts | 14 ++++- .../local-databases/database-manager.ts | 8 +-- extensions/ql-vscode/src/extension.ts | 51 ++++++++++++------- .../src/local-queries/local-queries.ts | 26 ++++++---- .../local-queries/local-databases.test.ts | 25 +++++---- 5 files changed, 78 insertions(+), 46 deletions(-) diff --git a/extensions/ql-vscode/src/common/logging/vscode/loggers.ts b/extensions/ql-vscode/src/common/logging/vscode/loggers.ts index 9cb953de6de..eee921210c1 100644 --- a/extensions/ql-vscode/src/common/logging/vscode/loggers.ts +++ b/extensions/ql-vscode/src/common/logging/vscode/loggers.ts @@ -11,8 +11,18 @@ export const extLogger = new OutputChannelLogger("CodeQL Extension Log"); export const queryServerLogger = new OutputChannelLogger("CodeQL Query Server"); // Logger for messages from the query server for warming overlay-base cache. -export const queryServerForWarmingOverlayBaseCacheLogger = - new OutputChannelLogger("CodeQL Query Server for warming overlay-base cache"); +let queryServerForWarmingOverlayBaseCacheLogger: + | OutputChannelLogger + | undefined; + +// construct queryServerForWarmingOverlayBaseCacheLogger lazily, as most users don't need it +export function getQueryServerForWarmingOverlayBaseCacheLogger(): OutputChannelLogger { + if (!queryServerForWarmingOverlayBaseCacheLogger) + queryServerForWarmingOverlayBaseCacheLogger = new OutputChannelLogger( + "CodeQL Query Server for warming overlay-base cache", + ); + return queryServerForWarmingOverlayBaseCacheLogger; +} // Logger for messages from the language server. export const languageServerLogger = new OutputChannelLogger( diff --git a/extensions/ql-vscode/src/databases/local-databases/database-manager.ts b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts index 516e9fd6fff..6847d1ad95a 100644 --- a/extensions/ql-vscode/src/databases/local-databases/database-manager.ts +++ b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts @@ -118,7 +118,6 @@ export class DatabaseManager extends DisposableObject { private readonly ctx: ExtensionContext, private readonly app: App, private readonly qs: QueryRunner, - private readonly qsForWarmingOverlayBaseCache: QueryRunner, private readonly cli: CodeQLCliServer, private readonly languageContext: LanguageContextStore, public logger: Logger, @@ -740,19 +739,20 @@ export class DatabaseManager extends DisposableObject { } } - public async withDatabaseInQsForWarmingOverlayBaseCache( + public async runWithDatabaseInSeparateQueryRunner( + queryRunner: QueryRunner, whatToDo: () => Promise, ) { try { if (this._currentDatabaseItem) { const dbItem = this._currentDatabaseItem; await this.qs.deregisterDatabase(dbItem); - await this.qsForWarmingOverlayBaseCache.registerDatabase(dbItem); + await queryRunner.registerDatabase(dbItem); } await whatToDo(); if (this._currentDatabaseItem) { const dbItem = this._currentDatabaseItem; - await this.qsForWarmingOverlayBaseCache.deregisterDatabase(dbItem); + await queryRunner.deregisterDatabase(dbItem); await this.qs.registerDatabase(dbItem); } } catch (e) { diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts index a3f8088b26a..c53516185f5 100644 --- a/extensions/ql-vscode/src/extension.ts +++ b/extensions/ql-vscode/src/extension.ts @@ -86,7 +86,7 @@ import { extLogger, languageServerLogger, queryServerLogger, - queryServerForWarmingOverlayBaseCacheLogger, + getQueryServerForWarmingOverlayBaseCacheLogger, } from "./common/logging/vscode"; import { QueryHistoryManager } from "./query-history/query-history-manager"; import type { CompletedLocalQueryInfo } from "./query-results"; @@ -173,7 +173,7 @@ function getCommands( app: App, cliServer: CodeQLCliServer, queryRunner: QueryRunner, - queryRunnerForWarmingOverlayBaseCache: QueryRunner, + queryRunnerForWarmingOverlayBaseCache: QueryRunner | undefined, languageClient: LanguageClient, ): BaseCommands { const getCliVersion = async () => { @@ -191,7 +191,9 @@ function getCommands( cliServer.restartCliServer(); await Promise.all([ queryRunner.restartQueryServer(progress), - queryRunnerForWarmingOverlayBaseCache.restartQueryServer(progress), + queryRunnerForWarmingOverlayBaseCache + ? queryRunnerForWarmingOverlayBaseCache.restartQueryServer(progress) + : {}, async () => { if (languageClient.isRunning()) { await languageClient.restart(); @@ -204,10 +206,12 @@ function getCommands( queryServerLogger, "CodeQL Query Server restarted.", ); - void showAndLogErrorMessage( - queryServerForWarmingOverlayBaseCacheLogger, - "CodeQL Query Server for warming overlay-base cache restarted.", - ); + if (queryRunnerForWarmingOverlayBaseCache) { + void showAndLogErrorMessage( + getQueryServerForWarmingOverlayBaseCacheLogger(), + "CodeQL Query Server for warming overlay-base cache restarted.", + ); + } }, { title: "Restarting Query Server", @@ -288,7 +292,7 @@ export interface CodeQLExtensionInterface { readonly ctx: ExtensionContext; readonly cliServer: CodeQLCliServer; readonly qs: QueryRunner; - readonly qsForWarmingOverlayBaseCache: QueryRunner; + readonly qsForWarmingOverlayBaseCache: QueryRunner | undefined; readonly distributionManager: DistributionManager; readonly databaseManager: DatabaseManager; readonly databaseUI: DatabaseUI; @@ -806,14 +810,24 @@ async function activateWithInstalledDistribution( false, ); - void extLogger.log("Initializing base cache warming query server client."); - const qsForWarmingOverlayBaseCache = await createQueryServer( - app, - qlConfigurationListener, - cliServer, - ctx, - true, - ); + let qsForWarmingOverlayBaseCache: QueryRunner | undefined; + + // construct qsForWarmingOverlayBaseCache lazily, as most users don't need it + async function getQsForWarmingOverlayBaseCache(): Promise { + if (!qsForWarmingOverlayBaseCache) { + void extLogger.log( + "Initializing base cache warming query server client.", + ); + qsForWarmingOverlayBaseCache = await createQueryServer( + app, + qlConfigurationListener, + cliServer, + ctx, + true, + ); + } + return qsForWarmingOverlayBaseCache; + } for (const glob of CLEAR_PACK_CACHE_ON_EDIT_GLOBS) { const fsWatcher = workspace.createFileSystemWatcher(glob); @@ -840,7 +854,6 @@ async function activateWithInstalledDistribution( ctx, app, qs, - qsForWarmingOverlayBaseCache, cliServer, languageContext, extLogger, @@ -1017,7 +1030,7 @@ async function activateWithInstalledDistribution( const localQueries = new LocalQueries( app, qs, - qsForWarmingOverlayBaseCache, + getQsForWarmingOverlayBaseCache, qhm, dbm, databaseFetcher, @@ -1309,7 +1322,7 @@ async function createQueryServer( ): Promise { const qsOpts = { logger: warmOverlayBaseCache - ? queryServerForWarmingOverlayBaseCacheLogger + ? getQueryServerForWarmingOverlayBaseCacheLogger() : queryServerLogger, contextStoragePath: getContextStoragePath(ctx), }; diff --git a/extensions/ql-vscode/src/local-queries/local-queries.ts b/extensions/ql-vscode/src/local-queries/local-queries.ts index def3572a3f3..428cead420a 100644 --- a/extensions/ql-vscode/src/local-queries/local-queries.ts +++ b/extensions/ql-vscode/src/local-queries/local-queries.ts @@ -74,7 +74,7 @@ export class LocalQueries extends DisposableObject { public constructor( private readonly app: ExtensionApp, private readonly queryRunner: QueryRunner, - private readonly queryRunnerForWarmingOverlayBaseCache: QueryRunner, + private readonly getQueryRunnerForWarmingOverlayBaseCache: () => Promise, private readonly queryHistoryManager: QueryHistoryManager, private readonly databaseManager: DatabaseManager, private readonly databaseFetcher: DatabaseFetcher, @@ -167,8 +167,10 @@ export class LocalQueries extends DisposableObject { private async runWarmOverlayBaseCacheForQuery( uri: Uri | undefined, ): Promise { - await this.databaseManager.withDatabaseInQsForWarmingOverlayBaseCache(() => - this.runQueryInternal(uri, true), + const queryRunner = await this.getQueryRunnerForWarmingOverlayBaseCache(); + await this.databaseManager.runWithDatabaseInSeparateQueryRunner( + queryRunner, + () => this.runQueryInternal(uri, true), ); } @@ -216,8 +218,10 @@ export class LocalQueries extends DisposableObject { private async runWarmOverlayBaseCacheForQueries( fileURIs: Uri[], ): Promise { - await this.databaseManager.withDatabaseInQsForWarmingOverlayBaseCache(() => - this.runQueriesInternal(fileURIs, true), + const queryRunner = await this.getQueryRunnerForWarmingOverlayBaseCache(); + await this.databaseManager.runWithDatabaseInSeparateQueryRunner( + queryRunner, + () => this.runQueriesInternal(fileURIs, true), ); } @@ -298,8 +302,10 @@ export class LocalQueries extends DisposableObject { private async runWarmOverlayBaseCacheForQuerySuite( fileUri: Uri, ): Promise { - await this.databaseManager.withDatabaseInQsForWarmingOverlayBaseCache(() => - this.runQuerySuiteInternal(fileUri, true), + const queryRunner = await this.getQueryRunnerForWarmingOverlayBaseCache(); + await this.databaseManager.runWithDatabaseInSeparateQueryRunner( + queryRunner, + () => this.runQuerySuiteInternal(fileUri, true), ); } @@ -341,7 +347,7 @@ export class LocalQueries extends DisposableObject { }); }); const queryRunner = warmOverlayBaseCache - ? this.queryRunnerForWarmingOverlayBaseCache + ? await this.getQueryRunnerForWarmingOverlayBaseCache() : this.queryRunner; const coreQueryRun = queryRunner.createQueryRun( databaseItem.databaseUri.fsPath, @@ -523,7 +529,7 @@ export class LocalQueries extends DisposableObject { await createTimestampFile(outputDir.querySaveDir); const queryRunner = warmOverlayBaseCache - ? this.queryRunnerForWarmingOverlayBaseCache + ? await this.getQueryRunnerForWarmingOverlayBaseCache() : this.queryRunner; if (queryRunner.customLogDirectory) { @@ -625,7 +631,7 @@ export class LocalQueries extends DisposableObject { const extensionPacks = await this.getDefaultExtensionPacks(additionalPacks); const queryRunner = warmOverlayBaseCache - ? this.queryRunnerForWarmingOverlayBaseCache + ? await this.getQueryRunnerForWarmingOverlayBaseCache() : this.queryRunner; const coreQueryRun = queryRunner.createQueryRun( databaseItem.databaseUri.fsPath, diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts index 516c2784861..517373bd3fd 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-queries/local-databases.test.ts @@ -39,6 +39,7 @@ import { LanguageContextStore } from "../../../../src/language-context-store"; describe("local databases", () => { let databaseManager: DatabaseManager; + let secondQueryRunner: QueryRunner; let extensionContext: ExtensionContext; let updateSpy: jest.Mock, []>; @@ -109,16 +110,6 @@ describe("local databases", () => { /**/ }, }), - mockedObject({ - registerDatabase: registerSpy2, - deregisterDatabase: deregisterSpy2, - onStart: () => { - /**/ - }, - onQueryRunStarting: () => { - /**/ - }, - }), mockedObject({ resolveDatabase: resolveDatabaseSpy, packAdd: packAddSpy, @@ -129,6 +120,17 @@ describe("local databases", () => { }), ); + secondQueryRunner = mockedObject({ + registerDatabase: registerSpy2, + deregisterDatabase: deregisterSpy2, + onStart: () => { + /**/ + }, + onQueryRunStarting: () => { + /**/ + }, + }); + // Unfortunately, during a test it is not possible to convert from // a single root workspace to multi-root, so must stub out relevant // functions @@ -350,7 +352,8 @@ describe("local databases", () => { // Should have registered this database expect(registerSpy).toHaveBeenCalledWith(mockDbItem); - await databaseManager.withDatabaseInQsForWarmingOverlayBaseCache( + await databaseManager.runWithDatabaseInSeparateQueryRunner( + secondQueryRunner, async () => { // Should have moved database registration expect(deregisterSpy).toHaveBeenCalledWith(mockDbItem);