From 6331cddbfd4608c9e3d4831e5d0dbb9574be694d Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 24 May 2023 16:51:38 +0200 Subject: [PATCH 01/11] Move DatabaseOptions to separate file --- .../ql-vscode/src/databases/local-databases.ts | 17 ++++------------- .../local-databases/database-options.ts | 12 ++++++++++++ 2 files changed, 16 insertions(+), 13 deletions(-) create mode 100644 extensions/ql-vscode/src/databases/local-databases/database-options.ts diff --git a/extensions/ql-vscode/src/databases/local-databases.ts b/extensions/ql-vscode/src/databases/local-databases.ts index 8bc50c70f41..400772b2a06 100644 --- a/extensions/ql-vscode/src/databases/local-databases.ts +++ b/extensions/ql-vscode/src/databases/local-databases.ts @@ -35,6 +35,10 @@ import { QlPackGenerator } from "../qlpack-generator"; import { QueryLanguage } from "../common/query-language"; import { App } from "../common/app"; import { existsSync } from "fs"; +import { + DatabaseOptions, + FullDatabaseOptions, +} from "./local-databases/database-options"; /** * databases.ts @@ -58,19 +62,6 @@ const CURRENT_DB = "currentDatabase"; */ const DB_LIST = "databaseList"; -export interface DatabaseOptions { - displayName?: string; - ignoreSourceArchive?: boolean; - dateAdded?: number | undefined; - language?: string; -} - -export interface FullDatabaseOptions extends DatabaseOptions { - ignoreSourceArchive: boolean; - dateAdded: number | undefined; - language: string | undefined; -} - interface PersistedDatabaseItem { uri: string; options?: DatabaseOptions; diff --git a/extensions/ql-vscode/src/databases/local-databases/database-options.ts b/extensions/ql-vscode/src/databases/local-databases/database-options.ts new file mode 100644 index 00000000000..b8990e7597a --- /dev/null +++ b/extensions/ql-vscode/src/databases/local-databases/database-options.ts @@ -0,0 +1,12 @@ +export interface DatabaseOptions { + displayName?: string; + ignoreSourceArchive?: boolean; + dateAdded?: number | undefined; + language?: string; +} + +export interface FullDatabaseOptions extends DatabaseOptions { + ignoreSourceArchive: boolean; + dateAdded: number | undefined; + language: string | undefined; +} From 4bb4627d30f09dcfe9fccaf118277a627207d132 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 24 May 2023 16:52:53 +0200 Subject: [PATCH 02/11] Move DatabaseItem to separate file --- .../src/databases/local-databases.ts | 96 ++----------------- .../local-databases/database-item.ts | 91 ++++++++++++++++++ 2 files changed, 97 insertions(+), 90 deletions(-) create mode 100644 extensions/ql-vscode/src/databases/local-databases/database-item.ts diff --git a/extensions/ql-vscode/src/databases/local-databases.ts b/extensions/ql-vscode/src/databases/local-databases.ts index 400772b2a06..97ae3dd3e52 100644 --- a/extensions/ql-vscode/src/databases/local-databases.ts +++ b/extensions/ql-vscode/src/databases/local-databases.ts @@ -35,10 +35,13 @@ import { QlPackGenerator } from "../qlpack-generator"; import { QueryLanguage } from "../common/query-language"; import { App } from "../common/app"; import { existsSync } from "fs"; +import { FullDatabaseOptions } from "./local-databases/database-options"; import { - DatabaseOptions, - FullDatabaseOptions, -} from "./local-databases/database-options"; + DatabaseItem, + PersistedDatabaseItem, +} from "./local-databases/database-item"; + +export { DatabaseItem } from "./local-databases/database-item"; /** * databases.ts @@ -62,11 +65,6 @@ const CURRENT_DB = "currentDatabase"; */ const DB_LIST = "databaseList"; -interface PersistedDatabaseItem { - uri: string; - options?: DatabaseOptions; -} - /** * The layout of the database. */ @@ -226,88 +224,6 @@ export class DatabaseResolver { } } -/** An item in the list of available databases */ -export interface DatabaseItem { - /** The URI of the database */ - readonly databaseUri: vscode.Uri; - /** The name of the database to be displayed in the UI */ - name: string; - - /** The primary language of the database or empty string if unknown */ - readonly language: string; - /** The URI of the database's source archive, or `undefined` if no source archive is to be used. */ - readonly sourceArchive: vscode.Uri | undefined; - /** - * The contents of the database. - * Will be `undefined` if the database is invalid. Can be updated by calling `refresh()`. - */ - readonly contents: DatabaseContents | undefined; - - /** - * The date this database was added as a unix timestamp. Or undefined if we don't know. - */ - readonly dateAdded: number | undefined; - - /** If the database is invalid, describes why. */ - readonly error: Error | undefined; - /** - * Resolves the contents of the database. - * - * @remarks - * The contents include the database directory, source archive, and metadata about the database. - * If the database is invalid, `this.error` is updated with the error object that describes why - * the database is invalid. This error is also thrown. - */ - refresh(): Promise; - /** - * Resolves a filename to its URI in the source archive. - * - * @param file Filename within the source archive. May be `undefined` to return a dummy file path. - */ - resolveSourceFile(file: string | undefined): vscode.Uri; - - /** - * Holds if the database item has a `.dbinfo` or `codeql-database.yml` file. - */ - hasMetadataFile(): Promise; - - /** - * Returns `sourceLocationPrefix` of exported database. - */ - getSourceLocationPrefix(server: cli.CodeQLCliServer): Promise; - - /** - * Returns dataset folder of exported database. - */ - getDatasetFolder(server: cli.CodeQLCliServer): Promise; - - /** - * Returns the root uri of the virtual filesystem for this database's source archive, - * as displayed in the filesystem explorer. - */ - getSourceArchiveExplorerUri(): vscode.Uri; - - /** - * Holds if `uri` belongs to this database's source archive. - */ - belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean; - - /** - * Whether the database may be affected by test execution for the given path. - */ - isAffectedByTest(testPath: string): Promise; - - /** - * Gets the state of this database, to be persisted in the workspace state. - */ - getPersistedState(): PersistedDatabaseItem; - - /** - * Verifies that this database item has a zipped source folder. Returns an error message if it does not. - */ - verifyZippedSources(): string | undefined; -} - export enum DatabaseEventKind { Add = "Add", Remove = "Remove", diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item.ts b/extensions/ql-vscode/src/databases/local-databases/database-item.ts new file mode 100644 index 00000000000..494ae3744c9 --- /dev/null +++ b/extensions/ql-vscode/src/databases/local-databases/database-item.ts @@ -0,0 +1,91 @@ +import vscode from "vscode"; +import * as cli from "../../codeql-cli/cli"; +import { DatabaseContents } from "../local-databases"; +import { DatabaseOptions } from "./database-options"; + +/** An item in the list of available databases */ +export interface DatabaseItem { + /** The URI of the database */ + readonly databaseUri: vscode.Uri; + /** The name of the database to be displayed in the UI */ + name: string; + + /** The primary language of the database or empty string if unknown */ + readonly language: string; + /** The URI of the database's source archive, or `undefined` if no source archive is to be used. */ + readonly sourceArchive: vscode.Uri | undefined; + /** + * The contents of the database. + * Will be `undefined` if the database is invalid. Can be updated by calling `refresh()`. + */ + readonly contents: DatabaseContents | undefined; + + /** + * The date this database was added as a unix timestamp. Or undefined if we don't know. + */ + readonly dateAdded: number | undefined; + + /** If the database is invalid, describes why. */ + readonly error: Error | undefined; + /** + * Resolves the contents of the database. + * + * @remarks + * The contents include the database directory, source archive, and metadata about the database. + * If the database is invalid, `this.error` is updated with the error object that describes why + * the database is invalid. This error is also thrown. + */ + refresh(): Promise; + /** + * Resolves a filename to its URI in the source archive. + * + * @param file Filename within the source archive. May be `undefined` to return a dummy file path. + */ + resolveSourceFile(file: string | undefined): vscode.Uri; + + /** + * Holds if the database item has a `.dbinfo` or `codeql-database.yml` file. + */ + hasMetadataFile(): Promise; + + /** + * Returns `sourceLocationPrefix` of exported database. + */ + getSourceLocationPrefix(server: cli.CodeQLCliServer): Promise; + + /** + * Returns dataset folder of exported database. + */ + getDatasetFolder(server: cli.CodeQLCliServer): Promise; + + /** + * Returns the root uri of the virtual filesystem for this database's source archive, + * as displayed in the filesystem explorer. + */ + getSourceArchiveExplorerUri(): vscode.Uri; + + /** + * Holds if `uri` belongs to this database's source archive. + */ + belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean; + + /** + * Whether the database may be affected by test execution for the given path. + */ + isAffectedByTest(testPath: string): Promise; + + /** + * Gets the state of this database, to be persisted in the workspace state. + */ + getPersistedState(): PersistedDatabaseItem; + + /** + * Verifies that this database item has a zipped source folder. Returns an error message if it does not. + */ + verifyZippedSources(): string | undefined; +} + +export interface PersistedDatabaseItem { + uri: string; + options?: DatabaseOptions; +} From d02e53fbd20a367a386e37d8320abfa0c6c1abef Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 24 May 2023 16:53:59 +0200 Subject: [PATCH 03/11] Move DatabaseItemImpl to separate file --- .../src/databases/local-databases.ts | 241 +---------------- .../local-databases/database-item-impl.ts | 247 ++++++++++++++++++ 2 files changed, 252 insertions(+), 236 deletions(-) create mode 100644 extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts diff --git a/extensions/ql-vscode/src/databases/local-databases.ts b/extensions/ql-vscode/src/databases/local-databases.ts index 97ae3dd3e52..c52f5ae0d68 100644 --- a/extensions/ql-vscode/src/databases/local-databases.ts +++ b/extensions/ql-vscode/src/databases/local-databases.ts @@ -1,30 +1,24 @@ -import { pathExists, stat, remove } from "fs-extra"; +import { pathExists, remove } from "fs-extra"; import { glob } from "glob"; -import { join, basename, resolve, relative, dirname, extname } from "path"; +import { join, basename, resolve, dirname, extname } from "path"; import * as vscode from "vscode"; import * as cli from "../codeql-cli/cli"; import { ExtensionContext } from "vscode"; import { showAndLogWarningMessage, showAndLogInformationMessage, - isLikelyDatabaseRoot, showAndLogExceptionWithTelemetry, isFolderAlreadyInWorkspace, getFirstWorkspaceFolder, showNeverAskAgainDialog, } from "../helpers"; import { ProgressCallback, withProgress } from "../common/vscode/progress"; -import { - zipArchiveScheme, - encodeArchiveBasePath, - decodeSourceArchiveUri, - encodeSourceArchiveUri, -} from "../common/vscode/archive-filesystem-provider"; +import { encodeArchiveBasePath } from "../common/vscode/archive-filesystem-provider"; import { DisposableObject } from "../pure/disposable-object"; import { Logger, extLogger } from "../common"; import { asError, getErrorMessage } from "../pure/helpers-pure"; import { QueryRunner } from "../query-server"; -import { containsPath, pathsEqual } from "../pure/files"; +import { containsPath } from "../pure/files"; import { redactableError } from "../pure/errors"; import { getAutogenerateQlPacks, @@ -40,6 +34,7 @@ import { DatabaseItem, PersistedDatabaseItem, } from "./local-databases/database-item"; +import { DatabaseItemImpl } from "./local-databases/database-item-impl"; export { DatabaseItem } from "./local-databases/database-item"; @@ -242,232 +237,6 @@ export interface DatabaseChangedEvent { item: DatabaseItem | undefined; } -// Exported for testing -export class DatabaseItemImpl implements DatabaseItem { - private _error: Error | undefined = undefined; - private _contents: DatabaseContents | undefined; - /** A cache of database info */ - private _dbinfo: cli.DbInfo | undefined; - - public constructor( - public readonly databaseUri: vscode.Uri, - contents: DatabaseContents | undefined, - private options: FullDatabaseOptions, - private readonly onChanged: (event: DatabaseChangedEvent) => void, - ) { - this._contents = contents; - } - - public get name(): string { - if (this.options.displayName) { - return this.options.displayName; - } else if (this._contents) { - return this._contents.name; - } else { - return basename(this.databaseUri.fsPath); - } - } - - public set name(newName: string) { - this.options.displayName = newName; - } - - public get sourceArchive(): vscode.Uri | undefined { - if (this.options.ignoreSourceArchive || this._contents === undefined) { - return undefined; - } else { - return this._contents.sourceArchiveUri; - } - } - - public get contents(): DatabaseContents | undefined { - return this._contents; - } - - public get dateAdded(): number | undefined { - return this.options.dateAdded; - } - - public get error(): Error | undefined { - return this._error; - } - - public async refresh(): Promise { - try { - try { - this._contents = await DatabaseResolver.resolveDatabaseContents( - this.databaseUri, - ); - this._error = undefined; - } catch (e) { - this._contents = undefined; - this._error = asError(e); - throw e; - } - } finally { - this.onChanged({ - kind: DatabaseEventKind.Refresh, - item: this, - }); - } - } - - public resolveSourceFile(uriStr: string | undefined): vscode.Uri { - const sourceArchive = this.sourceArchive; - const uri = uriStr ? vscode.Uri.parse(uriStr, true) : undefined; - if (uri && uri.scheme !== "file") { - throw new Error( - `Invalid uri scheme in ${uriStr}. Only 'file' is allowed.`, - ); - } - if (!sourceArchive) { - if (uri) { - return uri; - } else { - return this.databaseUri; - } - } - - if (uri) { - const relativeFilePath = decodeURI(uri.path) - .replace(":", "_") - .replace(/^\/*/, ""); - if (sourceArchive.scheme === zipArchiveScheme) { - const zipRef = decodeSourceArchiveUri(sourceArchive); - const pathWithinSourceArchive = - zipRef.pathWithinSourceArchive === "/" - ? relativeFilePath - : `${zipRef.pathWithinSourceArchive}/${relativeFilePath}`; - return encodeSourceArchiveUri({ - pathWithinSourceArchive, - sourceArchiveZipPath: zipRef.sourceArchiveZipPath, - }); - } else { - let newPath = sourceArchive.path; - if (!newPath.endsWith("/")) { - // Ensure a trailing slash. - newPath += "/"; - } - newPath += relativeFilePath; - - return sourceArchive.with({ path: newPath }); - } - } else { - return sourceArchive; - } - } - - /** - * Gets the state of this database, to be persisted in the workspace state. - */ - public getPersistedState(): PersistedDatabaseItem { - return { - uri: this.databaseUri.toString(true), - options: this.options, - }; - } - - /** - * Holds if the database item refers to an exported snapshot - */ - public async hasMetadataFile(): Promise { - return await isLikelyDatabaseRoot(this.databaseUri.fsPath); - } - - /** - * Returns information about a database. - */ - private async getDbInfo(server: cli.CodeQLCliServer): Promise { - if (this._dbinfo === undefined) { - this._dbinfo = await server.resolveDatabase(this.databaseUri.fsPath); - } - return this._dbinfo; - } - - /** - * Returns `sourceLocationPrefix` of database. Requires that the database - * has a `.dbinfo` file, which is the source of the prefix. - */ - public async getSourceLocationPrefix( - server: cli.CodeQLCliServer, - ): Promise { - const dbInfo = await this.getDbInfo(server); - return dbInfo.sourceLocationPrefix; - } - - /** - * Returns path to dataset folder of database. - */ - public async getDatasetFolder(server: cli.CodeQLCliServer): Promise { - const dbInfo = await this.getDbInfo(server); - return dbInfo.datasetFolder; - } - - public get language() { - return this.options.language || ""; - } - - /** - * Returns the root uri of the virtual filesystem for this database's source archive. - */ - public getSourceArchiveExplorerUri(): vscode.Uri { - const sourceArchive = this.sourceArchive; - if (sourceArchive === undefined || !sourceArchive.fsPath.endsWith(".zip")) { - throw new Error(this.verifyZippedSources()); - } - return encodeArchiveBasePath(sourceArchive.fsPath); - } - - public verifyZippedSources(): string | undefined { - const sourceArchive = this.sourceArchive; - if (sourceArchive === undefined) { - return `${this.name} has no source archive.`; - } - - if (!sourceArchive.fsPath.endsWith(".zip")) { - return `${this.name} has a source folder that is unzipped.`; - } - return; - } - - /** - * Holds if `uri` belongs to this database's source archive. - */ - public belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean { - if (this.sourceArchive === undefined) return false; - return ( - uri.scheme === zipArchiveScheme && - decodeSourceArchiveUri(uri).sourceArchiveZipPath === - this.sourceArchive.fsPath - ); - } - - public async isAffectedByTest(testPath: string): Promise { - const databasePath = this.databaseUri.fsPath; - if (!databasePath.endsWith(".testproj")) { - return false; - } - try { - const stats = await stat(testPath); - if (stats.isDirectory()) { - return !relative(testPath, databasePath).startsWith(".."); - } else { - // database for /one/two/three/test.ql is at /one/two/three/three.testproj - const testdir = dirname(testPath); - const testdirbase = basename(testdir); - return pathsEqual( - databasePath, - join(testdir, `${testdirbase}.testproj`), - process.platform, - ); - } - } catch { - // No information available for test path - assume database is unaffected. - return false; - } - } -} - /** * A promise that resolves to an event's result value when the event * `event` fires. If waiting for the event takes too long (by default diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts new file mode 100644 index 00000000000..460db694008 --- /dev/null +++ b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts @@ -0,0 +1,247 @@ +// Exported for testing +import * as cli from "../../codeql-cli/cli"; +import vscode from "vscode"; +import { FullDatabaseOptions } from "./database-options"; +import { basename, dirname, join, relative } from "path"; +import { asError } from "../../pure/helpers-pure"; +import { + decodeSourceArchiveUri, + encodeArchiveBasePath, + encodeSourceArchiveUri, + zipArchiveScheme, +} from "../../common/vscode/archive-filesystem-provider"; +import { DatabaseItem, PersistedDatabaseItem } from "./database-item"; +import { isLikelyDatabaseRoot } from "../../helpers"; +import { stat } from "fs-extra"; +import { pathsEqual } from "../../pure/files"; +import { + DatabaseChangedEvent, + DatabaseContents, + DatabaseEventKind, + DatabaseResolver, +} from "../local-databases"; + +export class DatabaseItemImpl implements DatabaseItem { + private _error: Error | undefined = undefined; + private _contents: DatabaseContents | undefined; + /** A cache of database info */ + private _dbinfo: cli.DbInfo | undefined; + + public constructor( + public readonly databaseUri: vscode.Uri, + contents: DatabaseContents | undefined, + private options: FullDatabaseOptions, + private readonly onChanged: (event: DatabaseChangedEvent) => void, + ) { + this._contents = contents; + } + + public get name(): string { + if (this.options.displayName) { + return this.options.displayName; + } else if (this._contents) { + return this._contents.name; + } else { + return basename(this.databaseUri.fsPath); + } + } + + public set name(newName: string) { + this.options.displayName = newName; + } + + public get sourceArchive(): vscode.Uri | undefined { + if (this.options.ignoreSourceArchive || this._contents === undefined) { + return undefined; + } else { + return this._contents.sourceArchiveUri; + } + } + + public get contents(): DatabaseContents | undefined { + return this._contents; + } + + public get dateAdded(): number | undefined { + return this.options.dateAdded; + } + + public get error(): Error | undefined { + return this._error; + } + + public async refresh(): Promise { + try { + try { + this._contents = await DatabaseResolver.resolveDatabaseContents( + this.databaseUri, + ); + this._error = undefined; + } catch (e) { + this._contents = undefined; + this._error = asError(e); + throw e; + } + } finally { + this.onChanged({ + kind: DatabaseEventKind.Refresh, + item: this, + }); + } + } + + public resolveSourceFile(uriStr: string | undefined): vscode.Uri { + const sourceArchive = this.sourceArchive; + const uri = uriStr ? vscode.Uri.parse(uriStr, true) : undefined; + if (uri && uri.scheme !== "file") { + throw new Error( + `Invalid uri scheme in ${uriStr}. Only 'file' is allowed.`, + ); + } + if (!sourceArchive) { + if (uri) { + return uri; + } else { + return this.databaseUri; + } + } + + if (uri) { + const relativeFilePath = decodeURI(uri.path) + .replace(":", "_") + .replace(/^\/*/, ""); + if (sourceArchive.scheme === zipArchiveScheme) { + const zipRef = decodeSourceArchiveUri(sourceArchive); + const pathWithinSourceArchive = + zipRef.pathWithinSourceArchive === "/" + ? relativeFilePath + : `${zipRef.pathWithinSourceArchive}/${relativeFilePath}`; + return encodeSourceArchiveUri({ + pathWithinSourceArchive, + sourceArchiveZipPath: zipRef.sourceArchiveZipPath, + }); + } else { + let newPath = sourceArchive.path; + if (!newPath.endsWith("/")) { + // Ensure a trailing slash. + newPath += "/"; + } + newPath += relativeFilePath; + + return sourceArchive.with({ path: newPath }); + } + } else { + return sourceArchive; + } + } + + /** + * Gets the state of this database, to be persisted in the workspace state. + */ + public getPersistedState(): PersistedDatabaseItem { + return { + uri: this.databaseUri.toString(true), + options: this.options, + }; + } + + /** + * Holds if the database item refers to an exported snapshot + */ + public async hasMetadataFile(): Promise { + return await isLikelyDatabaseRoot(this.databaseUri.fsPath); + } + + /** + * Returns information about a database. + */ + private async getDbInfo(server: cli.CodeQLCliServer): Promise { + if (this._dbinfo === undefined) { + this._dbinfo = await server.resolveDatabase(this.databaseUri.fsPath); + } + return this._dbinfo; + } + + /** + * Returns `sourceLocationPrefix` of database. Requires that the database + * has a `.dbinfo` file, which is the source of the prefix. + */ + public async getSourceLocationPrefix( + server: cli.CodeQLCliServer, + ): Promise { + const dbInfo = await this.getDbInfo(server); + return dbInfo.sourceLocationPrefix; + } + + /** + * Returns path to dataset folder of database. + */ + public async getDatasetFolder(server: cli.CodeQLCliServer): Promise { + const dbInfo = await this.getDbInfo(server); + return dbInfo.datasetFolder; + } + + public get language() { + return this.options.language || ""; + } + + /** + * Returns the root uri of the virtual filesystem for this database's source archive. + */ + public getSourceArchiveExplorerUri(): vscode.Uri { + const sourceArchive = this.sourceArchive; + if (sourceArchive === undefined || !sourceArchive.fsPath.endsWith(".zip")) { + throw new Error(this.verifyZippedSources()); + } + return encodeArchiveBasePath(sourceArchive.fsPath); + } + + public verifyZippedSources(): string | undefined { + const sourceArchive = this.sourceArchive; + if (sourceArchive === undefined) { + return `${this.name} has no source archive.`; + } + + if (!sourceArchive.fsPath.endsWith(".zip")) { + return `${this.name} has a source folder that is unzipped.`; + } + return; + } + + /** + * Holds if `uri` belongs to this database's source archive. + */ + public belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean { + if (this.sourceArchive === undefined) return false; + return ( + uri.scheme === zipArchiveScheme && + decodeSourceArchiveUri(uri).sourceArchiveZipPath === + this.sourceArchive.fsPath + ); + } + + public async isAffectedByTest(testPath: string): Promise { + const databasePath = this.databaseUri.fsPath; + if (!databasePath.endsWith(".testproj")) { + return false; + } + try { + const stats = await stat(testPath); + if (stats.isDirectory()) { + return !relative(testPath, databasePath).startsWith(".."); + } else { + // database for /one/two/three/test.ql is at /one/two/three/three.testproj + const testdir = dirname(testPath); + const testdirbase = basename(testdir); + return pathsEqual( + databasePath, + join(testdir, `${testdirbase}.testproj`), + process.platform, + ); + } + } catch { + // No information available for test path - assume database is unaffected. + return false; + } + } +} From 67983c64caa381555c016999ce72902437f28991 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 24 May 2023 16:55:31 +0200 Subject: [PATCH 04/11] Move DatabaseContents to separate file --- .../src/databases/local-databases.ts | 35 ++++--------------- .../local-databases/database-contents.ts | 30 ++++++++++++++++ .../local-databases/database-item-impl.ts | 2 +- .../local-databases/database-item.ts | 2 +- 4 files changed, 38 insertions(+), 31 deletions(-) create mode 100644 extensions/ql-vscode/src/databases/local-databases/database-contents.ts diff --git a/extensions/ql-vscode/src/databases/local-databases.ts b/extensions/ql-vscode/src/databases/local-databases.ts index c52f5ae0d68..abc5f1e6167 100644 --- a/extensions/ql-vscode/src/databases/local-databases.ts +++ b/extensions/ql-vscode/src/databases/local-databases.ts @@ -35,7 +35,13 @@ import { PersistedDatabaseItem, } from "./local-databases/database-item"; import { DatabaseItemImpl } from "./local-databases/database-item-impl"; +import { + DatabaseContents, + DatabaseContentsWithDbScheme, + DatabaseKind, +} from "./local-databases/database-contents"; +export { DatabaseContentsWithDbScheme } from "./local-databases/database-contents"; export { DatabaseItem } from "./local-databases/database-item"; /** @@ -60,35 +66,6 @@ const CURRENT_DB = "currentDatabase"; */ const DB_LIST = "databaseList"; -/** - * The layout of the database. - */ -export enum DatabaseKind { - /** A CodeQL database */ - Database, - /** A raw QL dataset */ - RawDataset, -} - -export interface DatabaseContents { - /** The layout of the database */ - kind: DatabaseKind; - /** - * The name of the database. - */ - name: string; - /** The URI of the QL dataset within the database. */ - datasetUri: vscode.Uri; - /** The URI of the source archive within the database, if one exists. */ - sourceArchiveUri?: vscode.Uri; - /** The URI of the CodeQL database scheme within the database, if exactly one exists. */ - dbSchemeUri?: vscode.Uri; -} - -export interface DatabaseContentsWithDbScheme extends DatabaseContents { - dbSchemeUri: vscode.Uri; // Always present -} - /** * An error thrown when we cannot find a valid database in a putative * database directory. diff --git a/extensions/ql-vscode/src/databases/local-databases/database-contents.ts b/extensions/ql-vscode/src/databases/local-databases/database-contents.ts new file mode 100644 index 00000000000..ce9f5d760e8 --- /dev/null +++ b/extensions/ql-vscode/src/databases/local-databases/database-contents.ts @@ -0,0 +1,30 @@ +import vscode from "vscode"; + +/** + * The layout of the database. + */ +export enum DatabaseKind { + /** A CodeQL database */ + Database, + /** A raw QL dataset */ + RawDataset, +} + +export interface DatabaseContents { + /** The layout of the database */ + kind: DatabaseKind; + /** + * The name of the database. + */ + name: string; + /** The URI of the QL dataset within the database. */ + datasetUri: vscode.Uri; + /** The URI of the source archive within the database, if one exists. */ + sourceArchiveUri?: vscode.Uri; + /** The URI of the CodeQL database scheme within the database, if exactly one exists. */ + dbSchemeUri?: vscode.Uri; +} + +export interface DatabaseContentsWithDbScheme extends DatabaseContents { + dbSchemeUri: vscode.Uri; // Always present +} diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts index 460db694008..7e9fc22c942 100644 --- a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts +++ b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts @@ -16,10 +16,10 @@ import { stat } from "fs-extra"; import { pathsEqual } from "../../pure/files"; import { DatabaseChangedEvent, - DatabaseContents, DatabaseEventKind, DatabaseResolver, } from "../local-databases"; +import { DatabaseContents } from "./database-contents"; export class DatabaseItemImpl implements DatabaseItem { private _error: Error | undefined = undefined; diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item.ts b/extensions/ql-vscode/src/databases/local-databases/database-item.ts index 494ae3744c9..0da295afbbc 100644 --- a/extensions/ql-vscode/src/databases/local-databases/database-item.ts +++ b/extensions/ql-vscode/src/databases/local-databases/database-item.ts @@ -1,6 +1,6 @@ import vscode from "vscode"; import * as cli from "../../codeql-cli/cli"; -import { DatabaseContents } from "../local-databases"; +import { DatabaseContents } from "./database-contents"; import { DatabaseOptions } from "./database-options"; /** An item in the list of available databases */ From 7888d210c48861c6f73d92d1b96e61051e74bbf3 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 24 May 2023 16:58:24 +0200 Subject: [PATCH 05/11] Move DatabaseResolver to separate file --- .../src/databases/local-databases.ts | 145 +----------------- .../local-databases/database-item-impl.ts | 7 +- .../local-databases/database-resolver.ts | 144 +++++++++++++++++ 3 files changed, 150 insertions(+), 146 deletions(-) create mode 100644 extensions/ql-vscode/src/databases/local-databases/database-resolver.ts diff --git a/extensions/ql-vscode/src/databases/local-databases.ts b/extensions/ql-vscode/src/databases/local-databases.ts index abc5f1e6167..d26849b8a5a 100644 --- a/extensions/ql-vscode/src/databases/local-databases.ts +++ b/extensions/ql-vscode/src/databases/local-databases.ts @@ -1,19 +1,15 @@ -import { pathExists, remove } from "fs-extra"; -import { glob } from "glob"; -import { join, basename, resolve, dirname, extname } from "path"; +import { remove } from "fs-extra"; +import { join, dirname, extname } from "path"; import * as vscode from "vscode"; import * as cli from "../codeql-cli/cli"; import { ExtensionContext } from "vscode"; import { - showAndLogWarningMessage, - showAndLogInformationMessage, showAndLogExceptionWithTelemetry, isFolderAlreadyInWorkspace, getFirstWorkspaceFolder, showNeverAskAgainDialog, } from "../helpers"; import { ProgressCallback, withProgress } from "../common/vscode/progress"; -import { encodeArchiveBasePath } from "../common/vscode/archive-filesystem-provider"; import { DisposableObject } from "../pure/disposable-object"; import { Logger, extLogger } from "../common"; import { asError, getErrorMessage } from "../pure/helpers-pure"; @@ -35,14 +31,11 @@ import { PersistedDatabaseItem, } from "./local-databases/database-item"; import { DatabaseItemImpl } from "./local-databases/database-item-impl"; -import { - DatabaseContents, - DatabaseContentsWithDbScheme, - DatabaseKind, -} from "./local-databases/database-contents"; +import { DatabaseResolver } from "./local-databases/database-resolver"; export { DatabaseContentsWithDbScheme } from "./local-databases/database-contents"; export { DatabaseItem } from "./local-databases/database-item"; +export { DatabaseResolver } from "./local-databases/database-resolver"; /** * databases.ts @@ -66,136 +59,6 @@ const CURRENT_DB = "currentDatabase"; */ const DB_LIST = "databaseList"; -/** - * An error thrown when we cannot find a valid database in a putative - * database directory. - */ -class InvalidDatabaseError extends Error {} - -async function findDataset(parentDirectory: string): Promise { - /* - * Look directly in the root - */ - let dbRelativePaths = await glob("db-*/", { - cwd: parentDirectory, - }); - - if (dbRelativePaths.length === 0) { - /* - * Check If they are in the old location - */ - dbRelativePaths = await glob("working/db-*/", { - cwd: parentDirectory, - }); - } - if (dbRelativePaths.length === 0) { - throw new InvalidDatabaseError( - `'${parentDirectory}' does not contain a dataset directory.`, - ); - } - - const dbAbsolutePath = join(parentDirectory, dbRelativePaths[0]); - if (dbRelativePaths.length > 1) { - void showAndLogWarningMessage( - `Found multiple dataset directories in database, using '${dbAbsolutePath}'.`, - ); - } - - return vscode.Uri.file(dbAbsolutePath); -} - -// exported for testing -export async function findSourceArchive( - databasePath: string, -): Promise { - const relativePaths = ["src", "output/src_archive"]; - - for (const relativePath of relativePaths) { - const basePath = join(databasePath, relativePath); - const zipPath = `${basePath}.zip`; - - // Prefer using a zip archive over a directory. - if (await pathExists(zipPath)) { - return encodeArchiveBasePath(zipPath); - } else if (await pathExists(basePath)) { - return vscode.Uri.file(basePath); - } - } - - void showAndLogInformationMessage( - `Could not find source archive for database '${databasePath}'. Assuming paths are absolute.`, - ); - return undefined; -} - -/** Gets the relative paths of all `.dbscheme` files in the given directory. */ -async function getDbSchemeFiles(dbDirectory: string): Promise { - return await glob("*.dbscheme", { cwd: dbDirectory }); -} - -export class DatabaseResolver { - public static async resolveDatabaseContents( - uri: vscode.Uri, - ): Promise { - if (uri.scheme !== "file") { - throw new Error( - `Database URI scheme '${uri.scheme}' not supported; only 'file' URIs are supported.`, - ); - } - const databasePath = uri.fsPath; - if (!(await pathExists(databasePath))) { - throw new InvalidDatabaseError( - `Database '${databasePath}' does not exist.`, - ); - } - - const contents = await this.resolveDatabase(databasePath); - - if (contents === undefined) { - throw new InvalidDatabaseError( - `'${databasePath}' is not a valid database.`, - ); - } - - // Look for a single dbscheme file within the database. - // This should be found in the dataset directory, regardless of the form of database. - const dbPath = contents.datasetUri.fsPath; - const dbSchemeFiles = await getDbSchemeFiles(dbPath); - if (dbSchemeFiles.length === 0) { - throw new InvalidDatabaseError( - `Database '${databasePath}' does not contain a CodeQL dbscheme under '${dbPath}'.`, - ); - } else if (dbSchemeFiles.length > 1) { - throw new InvalidDatabaseError( - `Database '${databasePath}' contains multiple CodeQL dbschemes under '${dbPath}'.`, - ); - } else { - const dbSchemeUri = vscode.Uri.file(resolve(dbPath, dbSchemeFiles[0])); - return { - ...contents, - dbSchemeUri, - }; - } - } - - public static async resolveDatabase( - databasePath: string, - ): Promise { - const name = basename(databasePath); - - // Look for dataset and source archive. - const datasetUri = await findDataset(databasePath); - const sourceArchiveUri = await findSourceArchive(databasePath); - - return { - kind: DatabaseKind.Database, - name, - datasetUri, - sourceArchiveUri, - }; - } -} - export enum DatabaseEventKind { Add = "Add", Remove = "Remove", diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts index 7e9fc22c942..3b239703840 100644 --- a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts +++ b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts @@ -14,12 +14,9 @@ import { DatabaseItem, PersistedDatabaseItem } from "./database-item"; import { isLikelyDatabaseRoot } from "../../helpers"; import { stat } from "fs-extra"; import { pathsEqual } from "../../pure/files"; -import { - DatabaseChangedEvent, - DatabaseEventKind, - DatabaseResolver, -} from "../local-databases"; +import { DatabaseChangedEvent, DatabaseEventKind } from "../local-databases"; import { DatabaseContents } from "./database-contents"; +import { DatabaseResolver } from "./database-resolver"; export class DatabaseItemImpl implements DatabaseItem { private _error: Error | undefined = undefined; diff --git a/extensions/ql-vscode/src/databases/local-databases/database-resolver.ts b/extensions/ql-vscode/src/databases/local-databases/database-resolver.ts new file mode 100644 index 00000000000..aa758c773d7 --- /dev/null +++ b/extensions/ql-vscode/src/databases/local-databases/database-resolver.ts @@ -0,0 +1,144 @@ +import vscode from "vscode"; +import { pathExists } from "fs-extra"; +import { basename, join, resolve } from "path"; +import { + DatabaseContents, + DatabaseContentsWithDbScheme, + DatabaseKind, +} from "./database-contents"; +import { glob } from "glob"; +import { + showAndLogInformationMessage, + showAndLogWarningMessage, +} from "../../helpers"; +import { encodeArchiveBasePath } from "../../common/vscode/archive-filesystem-provider"; + +export class DatabaseResolver { + public static async resolveDatabaseContents( + uri: vscode.Uri, + ): Promise { + if (uri.scheme !== "file") { + throw new Error( + `Database URI scheme '${uri.scheme}' not supported; only 'file' URIs are supported.`, + ); + } + const databasePath = uri.fsPath; + if (!(await pathExists(databasePath))) { + throw new InvalidDatabaseError( + `Database '${databasePath}' does not exist.`, + ); + } + + const contents = await this.resolveDatabase(databasePath); + + if (contents === undefined) { + throw new InvalidDatabaseError( + `'${databasePath}' is not a valid database.`, + ); + } + + // Look for a single dbscheme file within the database. + // This should be found in the dataset directory, regardless of the form of database. + const dbPath = contents.datasetUri.fsPath; + const dbSchemeFiles = await getDbSchemeFiles(dbPath); + if (dbSchemeFiles.length === 0) { + throw new InvalidDatabaseError( + `Database '${databasePath}' does not contain a CodeQL dbscheme under '${dbPath}'.`, + ); + } else if (dbSchemeFiles.length > 1) { + throw new InvalidDatabaseError( + `Database '${databasePath}' contains multiple CodeQL dbschemes under '${dbPath}'.`, + ); + } else { + const dbSchemeUri = vscode.Uri.file(resolve(dbPath, dbSchemeFiles[0])); + return { + ...contents, + dbSchemeUri, + }; + } + } + + public static async resolveDatabase( + databasePath: string, + ): Promise { + const name = basename(databasePath); + + // Look for dataset and source archive. + const datasetUri = await findDataset(databasePath); + const sourceArchiveUri = await findSourceArchive(databasePath); + + return { + kind: DatabaseKind.Database, + name, + datasetUri, + sourceArchiveUri, + }; + } +} + +/** + * An error thrown when we cannot find a valid database in a putative + * database directory. + */ +class InvalidDatabaseError extends Error {} + +async function findDataset(parentDirectory: string): Promise { + /* + * Look directly in the root + */ + let dbRelativePaths = await glob("db-*/", { + cwd: parentDirectory, + }); + + if (dbRelativePaths.length === 0) { + /* + * Check If they are in the old location + */ + dbRelativePaths = await glob("working/db-*/", { + cwd: parentDirectory, + }); + } + if (dbRelativePaths.length === 0) { + throw new InvalidDatabaseError( + `'${parentDirectory}' does not contain a dataset directory.`, + ); + } + + const dbAbsolutePath = join(parentDirectory, dbRelativePaths[0]); + if (dbRelativePaths.length > 1) { + void showAndLogWarningMessage( + `Found multiple dataset directories in database, using '${dbAbsolutePath}'.`, + ); + } + + return vscode.Uri.file(dbAbsolutePath); +} + +/** Gets the relative paths of all `.dbscheme` files in the given directory. */ +async function getDbSchemeFiles(dbDirectory: string): Promise { + return await glob("*.dbscheme", { cwd: dbDirectory }); +} + +// exported for testing +export async function findSourceArchive( + databasePath: string, +): Promise { + const relativePaths = ["src", "output/src_archive"]; + + for (const relativePath of relativePaths) { + const basePath = join(databasePath, relativePath); + const zipPath = `${basePath}.zip`; + + // Prefer using a zip archive over a directory. + if (await pathExists(zipPath)) { + return encodeArchiveBasePath(zipPath); + } else if (await pathExists(basePath)) { + return vscode.Uri.file(basePath); + } + } + + void showAndLogInformationMessage( + `Could not find source archive for database '${databasePath}'. Assuming paths are absolute.`, + ); + return undefined; +} From 59482c2b2c65feb764c9cdce280993c8d5c096dd Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 24 May 2023 17:00:52 +0200 Subject: [PATCH 06/11] Move DatabaseChangedEvent to separate file --- .../src/databases/local-databases.ts | 26 ++++++------------- .../local-databases/database-events.ts | 19 ++++++++++++++ .../local-databases/database-item-impl.ts | 2 +- 3 files changed, 28 insertions(+), 19 deletions(-) create mode 100644 extensions/ql-vscode/src/databases/local-databases/database-events.ts diff --git a/extensions/ql-vscode/src/databases/local-databases.ts b/extensions/ql-vscode/src/databases/local-databases.ts index d26849b8a5a..9d306c529ff 100644 --- a/extensions/ql-vscode/src/databases/local-databases.ts +++ b/extensions/ql-vscode/src/databases/local-databases.ts @@ -32,8 +32,16 @@ import { } from "./local-databases/database-item"; import { DatabaseItemImpl } from "./local-databases/database-item-impl"; import { DatabaseResolver } from "./local-databases/database-resolver"; +import { + DatabaseChangedEvent, + DatabaseEventKind, +} from "./local-databases/database-events"; export { DatabaseContentsWithDbScheme } from "./local-databases/database-contents"; +export { + DatabaseChangedEvent, + DatabaseEventKind, +} from "./local-databases/database-events"; export { DatabaseItem } from "./local-databases/database-item"; export { DatabaseResolver } from "./local-databases/database-resolver"; @@ -59,24 +67,6 @@ const CURRENT_DB = "currentDatabase"; */ const DB_LIST = "databaseList"; -export enum DatabaseEventKind { - Add = "Add", - Remove = "Remove", - - // Fired when databases are refreshed from persisted state - Refresh = "Refresh", - - // Fired when the current database changes - Change = "Change", - - Rename = "Rename", -} - -export interface DatabaseChangedEvent { - kind: DatabaseEventKind; - item: DatabaseItem | undefined; -} - /** * A promise that resolves to an event's result value when the event * `event` fires. If waiting for the event takes too long (by default diff --git a/extensions/ql-vscode/src/databases/local-databases/database-events.ts b/extensions/ql-vscode/src/databases/local-databases/database-events.ts new file mode 100644 index 00000000000..a48766ffbcb --- /dev/null +++ b/extensions/ql-vscode/src/databases/local-databases/database-events.ts @@ -0,0 +1,19 @@ +import { DatabaseItem } from "./database-item"; + +export enum DatabaseEventKind { + Add = "Add", + Remove = "Remove", + + // Fired when databases are refreshed from persisted state + Refresh = "Refresh", + + // Fired when the current database changes + Change = "Change", + + Rename = "Rename", +} + +export interface DatabaseChangedEvent { + kind: DatabaseEventKind; + item: DatabaseItem | undefined; +} diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts index 3b239703840..bd2c76786d8 100644 --- a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts +++ b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts @@ -14,9 +14,9 @@ import { DatabaseItem, PersistedDatabaseItem } from "./database-item"; import { isLikelyDatabaseRoot } from "../../helpers"; import { stat } from "fs-extra"; import { pathsEqual } from "../../pure/files"; -import { DatabaseChangedEvent, DatabaseEventKind } from "../local-databases"; import { DatabaseContents } from "./database-contents"; import { DatabaseResolver } from "./database-resolver"; +import { DatabaseChangedEvent, DatabaseEventKind } from "./database-events"; export class DatabaseItemImpl implements DatabaseItem { private _error: Error | undefined = undefined; From 60cfc311e5e895f88663e3d2138919b7d57acca2 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 24 May 2023 17:02:39 +0200 Subject: [PATCH 07/11] Move DatabaseManager to separate file --- .../src/databases/local-databases.ts | 655 +----------------- .../local-databases/database-manager.ts | 647 +++++++++++++++++ 2 files changed, 649 insertions(+), 653 deletions(-) create mode 100644 extensions/ql-vscode/src/databases/local-databases/database-manager.ts diff --git a/extensions/ql-vscode/src/databases/local-databases.ts b/extensions/ql-vscode/src/databases/local-databases.ts index 9d306c529ff..194286b30a8 100644 --- a/extensions/ql-vscode/src/databases/local-databases.ts +++ b/extensions/ql-vscode/src/databases/local-databases.ts @@ -1,41 +1,5 @@ -import { remove } from "fs-extra"; -import { join, dirname, extname } from "path"; +import { dirname } from "path"; import * as vscode from "vscode"; -import * as cli from "../codeql-cli/cli"; -import { ExtensionContext } from "vscode"; -import { - showAndLogExceptionWithTelemetry, - isFolderAlreadyInWorkspace, - getFirstWorkspaceFolder, - showNeverAskAgainDialog, -} from "../helpers"; -import { ProgressCallback, withProgress } from "../common/vscode/progress"; -import { DisposableObject } from "../pure/disposable-object"; -import { Logger, extLogger } from "../common"; -import { asError, getErrorMessage } from "../pure/helpers-pure"; -import { QueryRunner } from "../query-server"; -import { containsPath } from "../pure/files"; -import { redactableError } from "../pure/errors"; -import { - getAutogenerateQlPacks, - isCodespacesTemplate, - setAutogenerateQlPacks, -} from "../config"; -import { QlPackGenerator } from "../qlpack-generator"; -import { QueryLanguage } from "../common/query-language"; -import { App } from "../common/app"; -import { existsSync } from "fs"; -import { FullDatabaseOptions } from "./local-databases/database-options"; -import { - DatabaseItem, - PersistedDatabaseItem, -} from "./local-databases/database-item"; -import { DatabaseItemImpl } from "./local-databases/database-item-impl"; -import { DatabaseResolver } from "./local-databases/database-resolver"; -import { - DatabaseChangedEvent, - DatabaseEventKind, -} from "./local-databases/database-events"; export { DatabaseContentsWithDbScheme } from "./local-databases/database-contents"; export { @@ -43,6 +7,7 @@ export { DatabaseEventKind, } from "./local-databases/database-events"; export { DatabaseItem } from "./local-databases/database-item"; +export { DatabaseManager } from "./local-databases/database-manager"; export { DatabaseResolver } from "./local-databases/database-resolver"; /** @@ -55,622 +20,6 @@ export { DatabaseResolver } from "./local-databases/database-resolver"; * `DatabaseManager` class below. */ -/** - * The name of the key in the workspaceState dictionary in which we - * persist the current database across sessions. - */ -const CURRENT_DB = "currentDatabase"; - -/** - * The name of the key in the workspaceState dictionary in which we - * persist the list of databases across sessions. - */ -const DB_LIST = "databaseList"; - -/** - * A promise that resolves to an event's result value when the event - * `event` fires. If waiting for the event takes too long (by default - * >1000ms) log a warning, and resolve to undefined. - */ -function eventFired( - event: vscode.Event, - timeoutMs = 1000, -): Promise { - return new Promise((res, _rej) => { - const timeout = setTimeout(() => { - void extLogger.log( - `Waiting for event ${event} timed out after ${timeoutMs}ms`, - ); - res(undefined); - dispose(); - }, timeoutMs); - const disposable = event((e) => { - res(e); - dispose(); - }); - function dispose() { - clearTimeout(timeout); - disposable.dispose(); - } - }); -} - -export class DatabaseManager extends DisposableObject { - private readonly _onDidChangeDatabaseItem = this.push( - new vscode.EventEmitter(), - ); - - readonly onDidChangeDatabaseItem = this._onDidChangeDatabaseItem.event; - - private readonly _onDidChangeCurrentDatabaseItem = this.push( - new vscode.EventEmitter(), - ); - readonly onDidChangeCurrentDatabaseItem = - this._onDidChangeCurrentDatabaseItem.event; - - private readonly _databaseItems: DatabaseItem[] = []; - private _currentDatabaseItem: DatabaseItem | undefined = undefined; - - constructor( - private readonly ctx: ExtensionContext, - private readonly app: App, - private readonly qs: QueryRunner, - private readonly cli: cli.CodeQLCliServer, - public logger: Logger, - ) { - super(); - - qs.onStart(this.reregisterDatabases.bind(this)); - } - - /** - * Creates a {@link DatabaseItem} for the specified database, and adds it to the list of open - * databases. - */ - public async openDatabase( - progress: ProgressCallback, - token: vscode.CancellationToken, - uri: vscode.Uri, - makeSelected = true, - displayName?: string, - isTutorialDatabase?: boolean, - ): Promise { - const databaseItem = await this.createDatabaseItem(uri, displayName); - - return await this.addExistingDatabaseItem( - databaseItem, - progress, - makeSelected, - token, - isTutorialDatabase, - ); - } - - /** - * Adds a {@link DatabaseItem} to the list of open databases, if that database is not already on - * the list. - * - * Typically, the item will have been created by {@link createOrOpenDatabaseItem} or {@link openDatabase}. - */ - public async addExistingDatabaseItem( - databaseItem: DatabaseItem, - progress: ProgressCallback, - makeSelected: boolean, - token: vscode.CancellationToken, - isTutorialDatabase?: boolean, - ): Promise { - const existingItem = this.findDatabaseItem(databaseItem.databaseUri); - if (existingItem !== undefined) { - if (makeSelected) { - await this.setCurrentDatabaseItem(existingItem); - } - return existingItem; - } - - await this.addDatabaseItem(progress, token, databaseItem); - if (makeSelected) { - await this.setCurrentDatabaseItem(databaseItem); - } - await this.addDatabaseSourceArchiveFolder(databaseItem); - - if (isCodespacesTemplate() && !isTutorialDatabase) { - await this.createSkeletonPacks(databaseItem); - } - - return databaseItem; - } - - /** - * Creates a {@link DatabaseItem} for the specified database, without adding it to the list of - * open databases. - */ - private async createDatabaseItem( - uri: vscode.Uri, - displayName: string | undefined, - ): Promise { - const contents = await DatabaseResolver.resolveDatabaseContents(uri); - // Ignore the source archive for QLTest databases by default. - const isQLTestDatabase = extname(uri.fsPath) === ".testproj"; - const fullOptions: FullDatabaseOptions = { - ignoreSourceArchive: isQLTestDatabase, - // If a displayName is not passed in, the basename of folder containing the database is used. - displayName, - dateAdded: Date.now(), - language: await this.getPrimaryLanguage(uri.fsPath), - }; - const databaseItem = new DatabaseItemImpl( - uri, - contents, - fullOptions, - (event) => { - this._onDidChangeDatabaseItem.fire(event); - }, - ); - - return databaseItem; - } - - /** - * If the specified database is already on the list of open databases, returns that database's - * {@link DatabaseItem}. Otherwise, creates a new {@link DatabaseItem} without adding it to the - * list of open databases. - * - * The {@link DatabaseItem} can be added to the list of open databases later, via {@link addExistingDatabaseItem}. - */ - public async createOrOpenDatabaseItem( - uri: vscode.Uri, - ): Promise { - const existingItem = this.findDatabaseItem(uri); - if (existingItem !== undefined) { - // Use the one we already have. - return existingItem; - } - - // We don't add this to the list automatically, but the user can add it later. - return this.createDatabaseItem(uri, undefined); - } - - public async createSkeletonPacks(databaseItem: DatabaseItem) { - if (databaseItem === undefined) { - void this.logger.log( - "Could not create QL pack because no database is selected. Please add a database.", - ); - return; - } - - if (databaseItem.language === "") { - void this.logger.log( - "Could not create skeleton QL pack because the selected database's language is not set.", - ); - return; - } - - const firstWorkspaceFolder = getFirstWorkspaceFolder(); - const folderName = `codeql-custom-queries-${databaseItem.language}`; - - if ( - existsSync(join(firstWorkspaceFolder, folderName)) || - isFolderAlreadyInWorkspace(folderName) - ) { - return; - } - - if (getAutogenerateQlPacks() === "never") { - return; - } - - const answer = await showNeverAskAgainDialog( - `We've noticed you don't have a CodeQL pack available to analyze this database. Can we set up a query pack for you?`, - ); - - if (answer === "No") { - return; - } - - if (answer === "No, and never ask me again") { - await setAutogenerateQlPacks("never"); - return; - } - - try { - const qlPackGenerator = new QlPackGenerator( - folderName, - databaseItem.language as QueryLanguage, - this.cli, - firstWorkspaceFolder, - ); - await qlPackGenerator.generate(); - } catch (e: unknown) { - void this.logger.log( - `Could not create skeleton QL pack: ${getErrorMessage(e)}`, - ); - } - } - - private async reregisterDatabases( - progress: ProgressCallback, - token: vscode.CancellationToken, - ) { - let completed = 0; - await Promise.all( - this._databaseItems.map(async (databaseItem) => { - await this.registerDatabase(progress, token, databaseItem); - completed++; - progress({ - maxStep: this._databaseItems.length, - step: completed, - message: "Re-registering databases", - }); - }), - ); - } - - public async addDatabaseSourceArchiveFolder(item: DatabaseItem) { - // The folder may already be in workspace state from a previous - // session. If not, add it. - const index = this.getDatabaseWorkspaceFolderIndex(item); - if (index === -1) { - // Add that filesystem as a folder to the current workspace. - // - // It's important that we add workspace folders to the end, - // rather than beginning of the list, because the first - // workspace folder is special; if it gets updated, the entire - // extension host is restarted. (cf. - // https://github.com/microsoft/vscode/blob/e0d2ed907d1b22808c56127678fb436d604586a7/src/vs/workbench/contrib/relauncher/browser/relauncher.contribution.ts#L209-L214) - // - // This is undesirable, as we might be adding and removing many - // workspace folders as the user adds and removes databases. - const end = (vscode.workspace.workspaceFolders || []).length; - - const msg = item.verifyZippedSources(); - if (msg) { - void extLogger.log(`Could not add source folder because ${msg}`); - return; - } - - const uri = item.getSourceArchiveExplorerUri(); - void extLogger.log( - `Adding workspace folder for ${item.name} source archive at index ${end}`, - ); - if ((vscode.workspace.workspaceFolders || []).length < 2) { - // Adding this workspace folder makes the workspace - // multi-root, which may surprise the user. Let them know - // we're doing this. - void vscode.window.showInformationMessage( - `Adding workspace folder for source archive of database ${item.name}.`, - ); - } - vscode.workspace.updateWorkspaceFolders(end, 0, { - name: `[${item.name} source archive]`, - uri, - }); - // vscode api documentation says we must to wait for this event - // between multiple `updateWorkspaceFolders` calls. - await eventFired(vscode.workspace.onDidChangeWorkspaceFolders); - } - } - - private async createDatabaseItemFromPersistedState( - progress: ProgressCallback, - token: vscode.CancellationToken, - state: PersistedDatabaseItem, - ): Promise { - let displayName: string | undefined = undefined; - let ignoreSourceArchive = false; - let dateAdded = undefined; - let language = undefined; - if (state.options) { - if (typeof state.options.displayName === "string") { - displayName = state.options.displayName; - } - if (typeof state.options.ignoreSourceArchive === "boolean") { - ignoreSourceArchive = state.options.ignoreSourceArchive; - } - if (typeof state.options.dateAdded === "number") { - dateAdded = state.options.dateAdded; - } - language = state.options.language; - } - - const dbBaseUri = vscode.Uri.parse(state.uri, true); - if (language === undefined) { - // we haven't been successful yet at getting the language. try again - language = await this.getPrimaryLanguage(dbBaseUri.fsPath); - } - - const fullOptions: FullDatabaseOptions = { - ignoreSourceArchive, - displayName, - dateAdded, - language, - }; - const item = new DatabaseItemImpl( - dbBaseUri, - undefined, - fullOptions, - (event) => { - this._onDidChangeDatabaseItem.fire(event); - }, - ); - - // Avoid persisting the database state after adding since that should happen only after - // all databases have been added. - await this.addDatabaseItem(progress, token, item, false); - return item; - } - - public async loadPersistedState(): Promise { - return withProgress(async (progress, token) => { - const currentDatabaseUri = - this.ctx.workspaceState.get(CURRENT_DB); - const databases = this.ctx.workspaceState.get( - DB_LIST, - [], - ); - let step = 0; - progress({ - maxStep: databases.length, - message: "Loading persisted databases", - step, - }); - try { - void this.logger.log( - `Found ${databases.length} persisted databases: ${databases - .map((db) => db.uri) - .join(", ")}`, - ); - for (const database of databases) { - progress({ - maxStep: databases.length, - message: `Loading ${database.options?.displayName || "databases"}`, - step: ++step, - }); - - const databaseItem = await this.createDatabaseItemFromPersistedState( - progress, - token, - database, - ); - try { - await databaseItem.refresh(); - await this.registerDatabase(progress, token, databaseItem); - if (currentDatabaseUri === database.uri) { - await this.setCurrentDatabaseItem(databaseItem, true); - } - void this.logger.log( - `Loaded database ${databaseItem.name} at URI ${database.uri}.`, - ); - } catch (e) { - // When loading from persisted state, leave invalid databases in the list. They will be - // marked as invalid, and cannot be set as the current database. - void this.logger.log( - `Error loading database ${database.uri}: ${e}.`, - ); - } - } - await this.updatePersistedDatabaseList(); - } catch (e) { - // database list had an unexpected type - nothing to be done? - void showAndLogExceptionWithTelemetry( - redactableError( - asError(e), - )`Database list loading failed: ${getErrorMessage(e)}`, - ); - } - - void this.logger.log("Finished loading persisted databases."); - }); - } - - public get databaseItems(): readonly DatabaseItem[] { - return this._databaseItems; - } - - public get currentDatabaseItem(): DatabaseItem | undefined { - return this._currentDatabaseItem; - } - - public async setCurrentDatabaseItem( - item: DatabaseItem | undefined, - skipRefresh = false, - ): Promise { - if (!skipRefresh && item !== undefined) { - await item.refresh(); // Will throw on invalid database. - } - if (this._currentDatabaseItem !== item) { - this._currentDatabaseItem = item; - this.updatePersistedCurrentDatabaseItem(); - - await this.app.commands.execute( - "setContext", - "codeQL.currentDatabaseItem", - item?.name, - ); - - this._onDidChangeCurrentDatabaseItem.fire({ - item, - kind: DatabaseEventKind.Change, - }); - } - } - - /** - * Returns the index of the workspace folder that corresponds to the source archive of `item` - * if there is one, and -1 otherwise. - */ - private getDatabaseWorkspaceFolderIndex(item: DatabaseItem): number { - return (vscode.workspace.workspaceFolders || []).findIndex((folder) => - item.belongsToSourceArchiveExplorerUri(folder.uri), - ); - } - - public findDatabaseItem(uri: vscode.Uri): DatabaseItem | undefined { - const uriString = uri.toString(true); - return this._databaseItems.find( - (item) => item.databaseUri.toString(true) === uriString, - ); - } - - public findDatabaseItemBySourceArchive( - uri: vscode.Uri, - ): DatabaseItem | undefined { - const uriString = uri.toString(true); - return this._databaseItems.find( - (item) => - item.sourceArchive && item.sourceArchive.toString(true) === uriString, - ); - } - - private async addDatabaseItem( - progress: ProgressCallback, - token: vscode.CancellationToken, - item: DatabaseItem, - updatePersistedState = true, - ) { - this._databaseItems.push(item); - - if (updatePersistedState) { - await this.updatePersistedDatabaseList(); - } - - // Add this database item to the allow-list - // Database items reconstituted from persisted state - // will not have their contents yet. - if (item.contents?.datasetUri) { - await this.registerDatabase(progress, token, item); - } - // note that we use undefined as the item in order to reset the entire tree - this._onDidChangeDatabaseItem.fire({ - item: undefined, - kind: DatabaseEventKind.Add, - }); - } - - public async renameDatabaseItem(item: DatabaseItem, newName: string) { - item.name = newName; - await this.updatePersistedDatabaseList(); - this._onDidChangeDatabaseItem.fire({ - // pass undefined so that the entire tree is rebuilt in order to re-sort - item: undefined, - kind: DatabaseEventKind.Rename, - }); - } - - public async removeDatabaseItem( - progress: ProgressCallback, - token: vscode.CancellationToken, - item: DatabaseItem, - ) { - if (this._currentDatabaseItem === item) { - this._currentDatabaseItem = undefined; - } - const index = this.databaseItems.findIndex( - (searchItem) => searchItem === item, - ); - if (index >= 0) { - this._databaseItems.splice(index, 1); - } - await this.updatePersistedDatabaseList(); - - // Delete folder from workspace, if it is still there - const folderIndex = (vscode.workspace.workspaceFolders || []).findIndex( - (folder) => item.belongsToSourceArchiveExplorerUri(folder.uri), - ); - if (folderIndex >= 0) { - void extLogger.log(`Removing workspace folder at index ${folderIndex}`); - vscode.workspace.updateWorkspaceFolders(folderIndex, 1); - } - - // Remove this database item from the allow-list - await this.deregisterDatabase(progress, token, item); - - // Delete folder from file system only if it is controlled by the extension - if (this.isExtensionControlledLocation(item.databaseUri)) { - void extLogger.log("Deleting database from filesystem."); - await remove(item.databaseUri.fsPath).then( - () => void extLogger.log(`Deleted '${item.databaseUri.fsPath}'`), - (e: unknown) => - void extLogger.log( - `Failed to delete '${ - item.databaseUri.fsPath - }'. Reason: ${getErrorMessage(e)}`, - ), - ); - } - - // note that we use undefined as the item in order to reset the entire tree - this._onDidChangeDatabaseItem.fire({ - item: undefined, - kind: DatabaseEventKind.Remove, - }); - } - - public async removeAllDatabases( - progress: ProgressCallback, - token: vscode.CancellationToken, - ) { - for (const item of this.databaseItems) { - await this.removeDatabaseItem(progress, token, item); - } - } - - private async deregisterDatabase( - progress: ProgressCallback, - token: vscode.CancellationToken, - dbItem: DatabaseItem, - ) { - try { - await this.qs.deregisterDatabase(progress, token, dbItem); - } catch (e) { - const message = getErrorMessage(e); - if (message === "Connection is disposed.") { - // This is expected if the query server is not running. - void extLogger.log( - `Could not de-register database '${dbItem.name}' because query server is not running.`, - ); - return; - } - throw e; - } - } - private async registerDatabase( - progress: ProgressCallback, - token: vscode.CancellationToken, - dbItem: DatabaseItem, - ) { - await this.qs.registerDatabase(progress, token, dbItem); - } - - private updatePersistedCurrentDatabaseItem(): void { - void this.ctx.workspaceState.update( - CURRENT_DB, - this._currentDatabaseItem - ? this._currentDatabaseItem.databaseUri.toString(true) - : undefined, - ); - } - - private async updatePersistedDatabaseList(): Promise { - await this.ctx.workspaceState.update( - DB_LIST, - this._databaseItems.map((item) => item.getPersistedState()), - ); - } - - private isExtensionControlledLocation(uri: vscode.Uri) { - const storageUri = this.ctx.storageUri || this.ctx.globalStorageUri; - if (storageUri) { - return containsPath(storageUri.fsPath, uri.fsPath, process.platform); - } - return false; - } - - private async getPrimaryLanguage(dbPath: string) { - const dbInfo = await this.cli.resolveDatabase(dbPath); - return dbInfo.languages?.[0] || ""; - } -} - /** * Get the set of directories containing upgrades, given a list of * scripts returned by the cli's upgrade resolution. diff --git a/extensions/ql-vscode/src/databases/local-databases/database-manager.ts b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts new file mode 100644 index 00000000000..1383b179e11 --- /dev/null +++ b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts @@ -0,0 +1,647 @@ +import vscode, { ExtensionContext } from "vscode"; +import { extLogger, Logger } from "../../common"; +import { DisposableObject } from "../../pure/disposable-object"; +import { App } from "../../common/app"; +import { QueryRunner } from "../../query-server"; +import * as cli from "../../codeql-cli/cli"; +import { ProgressCallback, withProgress } from "../../common/vscode/progress"; +import { + getAutogenerateQlPacks, + isCodespacesTemplate, + setAutogenerateQlPacks, +} from "../../config"; +import { extname, join } from "path"; +import { FullDatabaseOptions } from "./database-options"; +import { DatabaseItemImpl } from "./database-item-impl"; +import { + getFirstWorkspaceFolder, + isFolderAlreadyInWorkspace, + showAndLogExceptionWithTelemetry, + showNeverAskAgainDialog, +} from "../../helpers"; +import { existsSync } from "fs"; +import { QlPackGenerator } from "../../qlpack-generator"; +import { QueryLanguage } from "../../common/query-language"; +import { asError, getErrorMessage } from "../../pure/helpers-pure"; +import { DatabaseItem, PersistedDatabaseItem } from "./database-item"; +import { redactableError } from "../../pure/errors"; +import { remove } from "fs-extra"; +import { containsPath } from "../../pure/files"; +import { DatabaseChangedEvent, DatabaseEventKind } from "./database-events"; +import { DatabaseResolver } from "./database-resolver"; + +/** + * The name of the key in the workspaceState dictionary in which we + * persist the current database across sessions. + */ +const CURRENT_DB = "currentDatabase"; + +/** + * The name of the key in the workspaceState dictionary in which we + * persist the list of databases across sessions. + */ +const DB_LIST = "databaseList"; + +/** + * A promise that resolves to an event's result value when the event + * `event` fires. If waiting for the event takes too long (by default + * >1000ms) log a warning, and resolve to undefined. + */ +function eventFired( + event: vscode.Event, + timeoutMs = 1000, +): Promise { + return new Promise((res, _rej) => { + const timeout = setTimeout(() => { + void extLogger.log( + `Waiting for event ${event} timed out after ${timeoutMs}ms`, + ); + res(undefined); + dispose(); + }, timeoutMs); + const disposable = event((e) => { + res(e); + dispose(); + }); + function dispose() { + clearTimeout(timeout); + disposable.dispose(); + } + }); +} + +export class DatabaseManager extends DisposableObject { + private readonly _onDidChangeDatabaseItem = this.push( + new vscode.EventEmitter(), + ); + + readonly onDidChangeDatabaseItem = this._onDidChangeDatabaseItem.event; + + private readonly _onDidChangeCurrentDatabaseItem = this.push( + new vscode.EventEmitter(), + ); + readonly onDidChangeCurrentDatabaseItem = + this._onDidChangeCurrentDatabaseItem.event; + + private readonly _databaseItems: DatabaseItem[] = []; + private _currentDatabaseItem: DatabaseItem | undefined = undefined; + + constructor( + private readonly ctx: ExtensionContext, + private readonly app: App, + private readonly qs: QueryRunner, + private readonly cli: cli.CodeQLCliServer, + public logger: Logger, + ) { + super(); + + qs.onStart(this.reregisterDatabases.bind(this)); + } + + /** + * Creates a {@link DatabaseItem} for the specified database, and adds it to the list of open + * databases. + */ + public async openDatabase( + progress: ProgressCallback, + token: vscode.CancellationToken, + uri: vscode.Uri, + makeSelected = true, + displayName?: string, + isTutorialDatabase?: boolean, + ): Promise { + const databaseItem = await this.createDatabaseItem(uri, displayName); + + return await this.addExistingDatabaseItem( + databaseItem, + progress, + makeSelected, + token, + isTutorialDatabase, + ); + } + + /** + * Adds a {@link DatabaseItem} to the list of open databases, if that database is not already on + * the list. + * + * Typically, the item will have been created by {@link createOrOpenDatabaseItem} or {@link openDatabase}. + */ + public async addExistingDatabaseItem( + databaseItem: DatabaseItem, + progress: ProgressCallback, + makeSelected: boolean, + token: vscode.CancellationToken, + isTutorialDatabase?: boolean, + ): Promise { + const existingItem = this.findDatabaseItem(databaseItem.databaseUri); + if (existingItem !== undefined) { + if (makeSelected) { + await this.setCurrentDatabaseItem(existingItem); + } + return existingItem; + } + + await this.addDatabaseItem(progress, token, databaseItem); + if (makeSelected) { + await this.setCurrentDatabaseItem(databaseItem); + } + await this.addDatabaseSourceArchiveFolder(databaseItem); + + if (isCodespacesTemplate() && !isTutorialDatabase) { + await this.createSkeletonPacks(databaseItem); + } + + return databaseItem; + } + + /** + * Creates a {@link DatabaseItem} for the specified database, without adding it to the list of + * open databases. + */ + private async createDatabaseItem( + uri: vscode.Uri, + displayName: string | undefined, + ): Promise { + const contents = await DatabaseResolver.resolveDatabaseContents(uri); + // Ignore the source archive for QLTest databases by default. + const isQLTestDatabase = extname(uri.fsPath) === ".testproj"; + const fullOptions: FullDatabaseOptions = { + ignoreSourceArchive: isQLTestDatabase, + // If a displayName is not passed in, the basename of folder containing the database is used. + displayName, + dateAdded: Date.now(), + language: await this.getPrimaryLanguage(uri.fsPath), + }; + const databaseItem = new DatabaseItemImpl( + uri, + contents, + fullOptions, + (event) => { + this._onDidChangeDatabaseItem.fire(event); + }, + ); + + return databaseItem; + } + + /** + * If the specified database is already on the list of open databases, returns that database's + * {@link DatabaseItem}. Otherwise, creates a new {@link DatabaseItem} without adding it to the + * list of open databases. + * + * The {@link DatabaseItem} can be added to the list of open databases later, via {@link addExistingDatabaseItem}. + */ + public async createOrOpenDatabaseItem( + uri: vscode.Uri, + ): Promise { + const existingItem = this.findDatabaseItem(uri); + if (existingItem !== undefined) { + // Use the one we already have. + return existingItem; + } + + // We don't add this to the list automatically, but the user can add it later. + return this.createDatabaseItem(uri, undefined); + } + + public async createSkeletonPacks(databaseItem: DatabaseItem) { + if (databaseItem === undefined) { + void this.logger.log( + "Could not create QL pack because no database is selected. Please add a database.", + ); + return; + } + + if (databaseItem.language === "") { + void this.logger.log( + "Could not create skeleton QL pack because the selected database's language is not set.", + ); + return; + } + + const firstWorkspaceFolder = getFirstWorkspaceFolder(); + const folderName = `codeql-custom-queries-${databaseItem.language}`; + + if ( + existsSync(join(firstWorkspaceFolder, folderName)) || + isFolderAlreadyInWorkspace(folderName) + ) { + return; + } + + if (getAutogenerateQlPacks() === "never") { + return; + } + + const answer = await showNeverAskAgainDialog( + `We've noticed you don't have a CodeQL pack available to analyze this database. Can we set up a query pack for you?`, + ); + + if (answer === "No") { + return; + } + + if (answer === "No, and never ask me again") { + await setAutogenerateQlPacks("never"); + return; + } + + try { + const qlPackGenerator = new QlPackGenerator( + folderName, + databaseItem.language as QueryLanguage, + this.cli, + firstWorkspaceFolder, + ); + await qlPackGenerator.generate(); + } catch (e: unknown) { + void this.logger.log( + `Could not create skeleton QL pack: ${getErrorMessage(e)}`, + ); + } + } + + private async reregisterDatabases( + progress: ProgressCallback, + token: vscode.CancellationToken, + ) { + let completed = 0; + await Promise.all( + this._databaseItems.map(async (databaseItem) => { + await this.registerDatabase(progress, token, databaseItem); + completed++; + progress({ + maxStep: this._databaseItems.length, + step: completed, + message: "Re-registering databases", + }); + }), + ); + } + + public async addDatabaseSourceArchiveFolder(item: DatabaseItem) { + // The folder may already be in workspace state from a previous + // session. If not, add it. + const index = this.getDatabaseWorkspaceFolderIndex(item); + if (index === -1) { + // Add that filesystem as a folder to the current workspace. + // + // It's important that we add workspace folders to the end, + // rather than beginning of the list, because the first + // workspace folder is special; if it gets updated, the entire + // extension host is restarted. (cf. + // https://github.com/microsoft/vscode/blob/e0d2ed907d1b22808c56127678fb436d604586a7/src/vs/workbench/contrib/relauncher/browser/relauncher.contribution.ts#L209-L214) + // + // This is undesirable, as we might be adding and removing many + // workspace folders as the user adds and removes databases. + const end = (vscode.workspace.workspaceFolders || []).length; + + const msg = item.verifyZippedSources(); + if (msg) { + void extLogger.log(`Could not add source folder because ${msg}`); + return; + } + + const uri = item.getSourceArchiveExplorerUri(); + void extLogger.log( + `Adding workspace folder for ${item.name} source archive at index ${end}`, + ); + if ((vscode.workspace.workspaceFolders || []).length < 2) { + // Adding this workspace folder makes the workspace + // multi-root, which may surprise the user. Let them know + // we're doing this. + void vscode.window.showInformationMessage( + `Adding workspace folder for source archive of database ${item.name}.`, + ); + } + vscode.workspace.updateWorkspaceFolders(end, 0, { + name: `[${item.name} source archive]`, + uri, + }); + // vscode api documentation says we must to wait for this event + // between multiple `updateWorkspaceFolders` calls. + await eventFired(vscode.workspace.onDidChangeWorkspaceFolders); + } + } + + private async createDatabaseItemFromPersistedState( + progress: ProgressCallback, + token: vscode.CancellationToken, + state: PersistedDatabaseItem, + ): Promise { + let displayName: string | undefined = undefined; + let ignoreSourceArchive = false; + let dateAdded = undefined; + let language = undefined; + if (state.options) { + if (typeof state.options.displayName === "string") { + displayName = state.options.displayName; + } + if (typeof state.options.ignoreSourceArchive === "boolean") { + ignoreSourceArchive = state.options.ignoreSourceArchive; + } + if (typeof state.options.dateAdded === "number") { + dateAdded = state.options.dateAdded; + } + language = state.options.language; + } + + const dbBaseUri = vscode.Uri.parse(state.uri, true); + if (language === undefined) { + // we haven't been successful yet at getting the language. try again + language = await this.getPrimaryLanguage(dbBaseUri.fsPath); + } + + const fullOptions: FullDatabaseOptions = { + ignoreSourceArchive, + displayName, + dateAdded, + language, + }; + const item = new DatabaseItemImpl( + dbBaseUri, + undefined, + fullOptions, + (event) => { + this._onDidChangeDatabaseItem.fire(event); + }, + ); + + // Avoid persisting the database state after adding since that should happen only after + // all databases have been added. + await this.addDatabaseItem(progress, token, item, false); + return item; + } + + public async loadPersistedState(): Promise { + return withProgress(async (progress, token) => { + const currentDatabaseUri = + this.ctx.workspaceState.get(CURRENT_DB); + const databases = this.ctx.workspaceState.get( + DB_LIST, + [], + ); + let step = 0; + progress({ + maxStep: databases.length, + message: "Loading persisted databases", + step, + }); + try { + void this.logger.log( + `Found ${databases.length} persisted databases: ${databases + .map((db) => db.uri) + .join(", ")}`, + ); + for (const database of databases) { + progress({ + maxStep: databases.length, + message: `Loading ${database.options?.displayName || "databases"}`, + step: ++step, + }); + + const databaseItem = await this.createDatabaseItemFromPersistedState( + progress, + token, + database, + ); + try { + await databaseItem.refresh(); + await this.registerDatabase(progress, token, databaseItem); + if (currentDatabaseUri === database.uri) { + await this.setCurrentDatabaseItem(databaseItem, true); + } + void this.logger.log( + `Loaded database ${databaseItem.name} at URI ${database.uri}.`, + ); + } catch (e) { + // When loading from persisted state, leave invalid databases in the list. They will be + // marked as invalid, and cannot be set as the current database. + void this.logger.log( + `Error loading database ${database.uri}: ${e}.`, + ); + } + } + await this.updatePersistedDatabaseList(); + } catch (e) { + // database list had an unexpected type - nothing to be done? + void showAndLogExceptionWithTelemetry( + redactableError( + asError(e), + )`Database list loading failed: ${getErrorMessage(e)}`, + ); + } + + void this.logger.log("Finished loading persisted databases."); + }); + } + + public get databaseItems(): readonly DatabaseItem[] { + return this._databaseItems; + } + + public get currentDatabaseItem(): DatabaseItem | undefined { + return this._currentDatabaseItem; + } + + public async setCurrentDatabaseItem( + item: DatabaseItem | undefined, + skipRefresh = false, + ): Promise { + if (!skipRefresh && item !== undefined) { + await item.refresh(); // Will throw on invalid database. + } + if (this._currentDatabaseItem !== item) { + this._currentDatabaseItem = item; + this.updatePersistedCurrentDatabaseItem(); + + await this.app.commands.execute( + "setContext", + "codeQL.currentDatabaseItem", + item?.name, + ); + + this._onDidChangeCurrentDatabaseItem.fire({ + item, + kind: DatabaseEventKind.Change, + }); + } + } + + /** + * Returns the index of the workspace folder that corresponds to the source archive of `item` + * if there is one, and -1 otherwise. + */ + private getDatabaseWorkspaceFolderIndex(item: DatabaseItem): number { + return (vscode.workspace.workspaceFolders || []).findIndex((folder) => + item.belongsToSourceArchiveExplorerUri(folder.uri), + ); + } + + public findDatabaseItem(uri: vscode.Uri): DatabaseItem | undefined { + const uriString = uri.toString(true); + return this._databaseItems.find( + (item) => item.databaseUri.toString(true) === uriString, + ); + } + + public findDatabaseItemBySourceArchive( + uri: vscode.Uri, + ): DatabaseItem | undefined { + const uriString = uri.toString(true); + return this._databaseItems.find( + (item) => + item.sourceArchive && item.sourceArchive.toString(true) === uriString, + ); + } + + private async addDatabaseItem( + progress: ProgressCallback, + token: vscode.CancellationToken, + item: DatabaseItem, + updatePersistedState = true, + ) { + this._databaseItems.push(item); + + if (updatePersistedState) { + await this.updatePersistedDatabaseList(); + } + + // Add this database item to the allow-list + // Database items reconstituted from persisted state + // will not have their contents yet. + if (item.contents?.datasetUri) { + await this.registerDatabase(progress, token, item); + } + // note that we use undefined as the item in order to reset the entire tree + this._onDidChangeDatabaseItem.fire({ + item: undefined, + kind: DatabaseEventKind.Add, + }); + } + + public async renameDatabaseItem(item: DatabaseItem, newName: string) { + item.name = newName; + await this.updatePersistedDatabaseList(); + this._onDidChangeDatabaseItem.fire({ + // pass undefined so that the entire tree is rebuilt in order to re-sort + item: undefined, + kind: DatabaseEventKind.Rename, + }); + } + + public async removeDatabaseItem( + progress: ProgressCallback, + token: vscode.CancellationToken, + item: DatabaseItem, + ) { + if (this._currentDatabaseItem === item) { + this._currentDatabaseItem = undefined; + } + const index = this.databaseItems.findIndex( + (searchItem) => searchItem === item, + ); + if (index >= 0) { + this._databaseItems.splice(index, 1); + } + await this.updatePersistedDatabaseList(); + + // Delete folder from workspace, if it is still there + const folderIndex = (vscode.workspace.workspaceFolders || []).findIndex( + (folder) => item.belongsToSourceArchiveExplorerUri(folder.uri), + ); + if (folderIndex >= 0) { + void extLogger.log(`Removing workspace folder at index ${folderIndex}`); + vscode.workspace.updateWorkspaceFolders(folderIndex, 1); + } + + // Remove this database item from the allow-list + await this.deregisterDatabase(progress, token, item); + + // Delete folder from file system only if it is controlled by the extension + if (this.isExtensionControlledLocation(item.databaseUri)) { + void extLogger.log("Deleting database from filesystem."); + await remove(item.databaseUri.fsPath).then( + () => void extLogger.log(`Deleted '${item.databaseUri.fsPath}'`), + (e: unknown) => + void extLogger.log( + `Failed to delete '${ + item.databaseUri.fsPath + }'. Reason: ${getErrorMessage(e)}`, + ), + ); + } + + // note that we use undefined as the item in order to reset the entire tree + this._onDidChangeDatabaseItem.fire({ + item: undefined, + kind: DatabaseEventKind.Remove, + }); + } + + public async removeAllDatabases( + progress: ProgressCallback, + token: vscode.CancellationToken, + ) { + for (const item of this.databaseItems) { + await this.removeDatabaseItem(progress, token, item); + } + } + + private async deregisterDatabase( + progress: ProgressCallback, + token: vscode.CancellationToken, + dbItem: DatabaseItem, + ) { + try { + await this.qs.deregisterDatabase(progress, token, dbItem); + } catch (e) { + const message = getErrorMessage(e); + if (message === "Connection is disposed.") { + // This is expected if the query server is not running. + void extLogger.log( + `Could not de-register database '${dbItem.name}' because query server is not running.`, + ); + return; + } + throw e; + } + } + private async registerDatabase( + progress: ProgressCallback, + token: vscode.CancellationToken, + dbItem: DatabaseItem, + ) { + await this.qs.registerDatabase(progress, token, dbItem); + } + + private updatePersistedCurrentDatabaseItem(): void { + void this.ctx.workspaceState.update( + CURRENT_DB, + this._currentDatabaseItem + ? this._currentDatabaseItem.databaseUri.toString(true) + : undefined, + ); + } + + private async updatePersistedDatabaseList(): Promise { + await this.ctx.workspaceState.update( + DB_LIST, + this._databaseItems.map((item) => item.getPersistedState()), + ); + } + + private isExtensionControlledLocation(uri: vscode.Uri) { + const storageUri = this.ctx.storageUri || this.ctx.globalStorageUri; + if (storageUri) { + return containsPath(storageUri.fsPath, uri.fsPath, process.platform); + } + return false; + } + + private async getPrimaryLanguage(dbPath: string) { + const dbInfo = await this.cli.resolveDatabase(dbPath); + return dbInfo.languages?.[0] || ""; + } +} From d608c057a4669bf1cdd249245625dc35c38ff505 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 24 May 2023 17:03:14 +0200 Subject: [PATCH 08/11] Remove unused getUpgradesDirectories function --- .../ql-vscode/src/databases/local-databases.ts | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/extensions/ql-vscode/src/databases/local-databases.ts b/extensions/ql-vscode/src/databases/local-databases.ts index 194286b30a8..038afb0563d 100644 --- a/extensions/ql-vscode/src/databases/local-databases.ts +++ b/extensions/ql-vscode/src/databases/local-databases.ts @@ -1,6 +1,3 @@ -import { dirname } from "path"; -import * as vscode from "vscode"; - export { DatabaseContentsWithDbScheme } from "./local-databases/database-contents"; export { DatabaseChangedEvent, @@ -19,15 +16,3 @@ export { DatabaseResolver } from "./local-databases/database-resolver"; * The source of truth of the current state resides inside the * `DatabaseManager` class below. */ - -/** - * Get the set of directories containing upgrades, given a list of - * scripts returned by the cli's upgrade resolution. - */ -export function getUpgradesDirectories(scripts: string[]): vscode.Uri[] { - const parentDirs = scripts.map((dir) => dirname(dir)); - const uniqueParentDirs = new Set(parentDirs); - return Array.from(uniqueParentDirs).map((filePath) => - vscode.Uri.file(filePath), - ); -} From 6b9c3491ec4324d6c2f65b9d31aeda261862487c Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 24 May 2023 17:04:14 +0200 Subject: [PATCH 09/11] Move local-databases.ts to local-databases/index.ts --- .../ql-vscode/src/databases/local-databases.ts | 18 ------------------ .../src/databases/local-databases/index.ts | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 18 deletions(-) delete mode 100644 extensions/ql-vscode/src/databases/local-databases.ts create mode 100644 extensions/ql-vscode/src/databases/local-databases/index.ts diff --git a/extensions/ql-vscode/src/databases/local-databases.ts b/extensions/ql-vscode/src/databases/local-databases.ts deleted file mode 100644 index 038afb0563d..00000000000 --- a/extensions/ql-vscode/src/databases/local-databases.ts +++ /dev/null @@ -1,18 +0,0 @@ -export { DatabaseContentsWithDbScheme } from "./local-databases/database-contents"; -export { - DatabaseChangedEvent, - DatabaseEventKind, -} from "./local-databases/database-events"; -export { DatabaseItem } from "./local-databases/database-item"; -export { DatabaseManager } from "./local-databases/database-manager"; -export { DatabaseResolver } from "./local-databases/database-resolver"; - -/** - * databases.ts - * ------------ - * Managing state of what the current database is, and what other - * databases have been recently selected. - * - * The source of truth of the current state resides inside the - * `DatabaseManager` class below. - */ diff --git a/extensions/ql-vscode/src/databases/local-databases/index.ts b/extensions/ql-vscode/src/databases/local-databases/index.ts new file mode 100644 index 00000000000..5a8e994dbec --- /dev/null +++ b/extensions/ql-vscode/src/databases/local-databases/index.ts @@ -0,0 +1,15 @@ +export { DatabaseContentsWithDbScheme } from "./database-contents"; +export { DatabaseChangedEvent, DatabaseEventKind } from "./database-events"; +export { DatabaseItem } from "./database-item"; +export { DatabaseManager } from "./database-manager"; +export { DatabaseResolver } from "./database-resolver"; + +/** + * databases.ts + * ------------ + * Managing state of what the current database is, and what other + * databases have been recently selected. + * + * The source of truth of the current state resides inside the + * `DatabaseManager` class below. + */ From c004f187208bb848ceb96cd45fe16cfc6565c3cf Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 24 May 2023 17:04:36 +0200 Subject: [PATCH 10/11] Remove outdated comment --- .../ql-vscode/src/databases/local-databases/index.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/extensions/ql-vscode/src/databases/local-databases/index.ts b/extensions/ql-vscode/src/databases/local-databases/index.ts index 5a8e994dbec..44c67a080e8 100644 --- a/extensions/ql-vscode/src/databases/local-databases/index.ts +++ b/extensions/ql-vscode/src/databases/local-databases/index.ts @@ -3,13 +3,3 @@ export { DatabaseChangedEvent, DatabaseEventKind } from "./database-events"; export { DatabaseItem } from "./database-item"; export { DatabaseManager } from "./database-manager"; export { DatabaseResolver } from "./database-resolver"; - -/** - * databases.ts - * ------------ - * Managing state of what the current database is, and what other - * databases have been recently selected. - * - * The source of truth of the current state resides inside the - * `DatabaseManager` class below. - */ From 070715560362875b9dd6ba2529ec43c5100d6fa6 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 24 May 2023 17:08:00 +0200 Subject: [PATCH 11/11] Fix imports for tests --- .../ql-vscode/src/databases/local-databases/index.ts | 8 +++++++- .../minimal-workspace/local-databases.test.ts | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/extensions/ql-vscode/src/databases/local-databases/index.ts b/extensions/ql-vscode/src/databases/local-databases/index.ts index 44c67a080e8..fbca66f647c 100644 --- a/extensions/ql-vscode/src/databases/local-databases/index.ts +++ b/extensions/ql-vscode/src/databases/local-databases/index.ts @@ -1,5 +1,11 @@ -export { DatabaseContentsWithDbScheme } from "./database-contents"; +export { + DatabaseContents, + DatabaseContentsWithDbScheme, + DatabaseKind, +} from "./database-contents"; export { DatabaseChangedEvent, DatabaseEventKind } from "./database-events"; export { DatabaseItem } from "./database-item"; +export { DatabaseItemImpl } from "./database-item-impl"; export { DatabaseManager } from "./database-manager"; export { DatabaseResolver } from "./database-resolver"; +export { DatabaseOptions, FullDatabaseOptions } from "./database-options"; diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-databases.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-databases.test.ts index 034c86f66d2..00bad35c242 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-databases.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-databases.test.ts @@ -9,7 +9,6 @@ import { DatabaseItemImpl, DatabaseManager, DatabaseResolver, - findSourceArchive, FullDatabaseOptions, } from "../../../src/databases/local-databases"; import { Logger } from "../../../src/common"; @@ -32,6 +31,7 @@ import { mockDbOptions, sourceLocationUri, } from "../../factories/databases/databases"; +import { findSourceArchive } from "../../../src/databases/local-databases/database-resolver"; describe("local databases", () => { let databaseManager: DatabaseManager;