diff --git a/extensions/ql-vscode/CHANGELOG.md b/extensions/ql-vscode/CHANGELOG.md index f42e7d87bb2..908a1248ed7 100644 --- a/extensions/ql-vscode/CHANGELOG.md +++ b/extensions/ql-vscode/CHANGELOG.md @@ -3,12 +3,12 @@ ## [UNRELEASED] - Fix a bug where invoking _View AST_ from the file explorer would not view the selected file. Instead it would view the active editor. Also, prevent the _View AST_ from appearing if the current selection includes a directory or multiple files. [#1113](https://github.com/github/vscode-codeql/pull/1113) +- Add query history items as soon as a query is run, including new icons for each history item. [#1094](https://github.com/github/vscode-codeql/pull/1094) ## 1.5.10 - 25 January 2022 - Fix a bug where the results view moved column even when it was already visible. [#1070](https://github.com/github/vscode-codeql/pull/1070) - Add packaging-related commands. _CodeQL: Download Packs_ downloads query packs from the package registry that can be run locally, and _CodeQL: Install Pack Dependencies_ installs dependencies for packs in your workspace. [#1076](https://github.com/github/vscode-codeql/pull/1076) -- Add query history items as soon as a query is run, including new icons for each history item. [#1094](https://github.com/github/vscode-codeql/pull/1094) ## 1.5.9 - 17 December 2021 diff --git a/extensions/ql-vscode/package-lock.json b/extensions/ql-vscode/package-lock.json index d47c5dcf1e6..e1a75874cbe 100644 --- a/extensions/ql-vscode/package-lock.json +++ b/extensions/ql-vscode/package-lock.json @@ -18,6 +18,7 @@ "glob-promise": "^3.4.0", "js-yaml": "^3.14.0", "minimist": "~1.2.5", + "nanoid": "^3.2.0", "node-fetch": "~2.6.7", "path-browserify": "^1.0.1", "react": "^17.0.2", @@ -53,6 +54,7 @@ "@types/js-yaml": "^3.12.5", "@types/jszip": "~3.1.6", "@types/mocha": "^9.0.0", + "@types/nanoid": "^3.0.0", "@types/node": "^12.14.1", "@types/node-fetch": "~2.5.2", "@types/proxyquire": "~1.3.28", @@ -1207,6 +1209,16 @@ "integrity": "sha512-scN0hAWyLVAvLR9AyW7HoFF5sJZglyBsbPuHO4fv7JRvfmPBMfp1ozWqOf/e4wwPNxezBZXRfWzMb6iFLgEVRA==", "dev": true }, + "node_modules/@types/nanoid": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/nanoid/-/nanoid-3.0.0.tgz", + "integrity": "sha512-UXitWSmXCwhDmAKe7D3hNQtQaHeHt5L8LO1CB8GF8jlYVzOv5cBWDNqiJ+oPEWrWei3i3dkZtHY/bUtd0R/uOQ==", + "deprecated": "This is a stub types definition. nanoid provides its own type definitions, so you do not need this installed.", + "dev": true, + "dependencies": { + "nanoid": "*" + } + }, "node_modules/@types/node": { "version": "12.19.4", "resolved": "https://registry.npmjs.org/@types/node/-/node-12.19.4.tgz", @@ -8381,6 +8393,18 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, + "node_modules/mocha/node_modules/nanoid": { + "version": "3.1.25", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.25.tgz", + "integrity": "sha512-rdwtIXaXCLFAQbnfqDRnI6jaRHp9fTcYBjtFKE8eezcZ7LuLjhUaQGNeMXf1HmRoCH32CLz6XwX0TtxEOS/A3Q==", + "dev": true, + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/mocha/node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -8574,10 +8598,9 @@ "optional": true }, "node_modules/nanoid": { - "version": "3.1.25", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.25.tgz", - "integrity": "sha512-rdwtIXaXCLFAQbnfqDRnI6jaRHp9fTcYBjtFKE8eezcZ7LuLjhUaQGNeMXf1HmRoCH32CLz6XwX0TtxEOS/A3Q==", - "dev": true, + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.2.0.tgz", + "integrity": "sha512-fmsZYa9lpn69Ad5eDn7FMcnnSR+8R34W9qJEijxYhTbfOWzr22n1QxCMzXLK+ODyW2973V3Fux959iQoUxzUIA==", "bin": { "nanoid": "bin/nanoid.cjs" }, @@ -14067,6 +14090,15 @@ "integrity": "sha512-scN0hAWyLVAvLR9AyW7HoFF5sJZglyBsbPuHO4fv7JRvfmPBMfp1ozWqOf/e4wwPNxezBZXRfWzMb6iFLgEVRA==", "dev": true }, + "@types/nanoid": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/nanoid/-/nanoid-3.0.0.tgz", + "integrity": "sha512-UXitWSmXCwhDmAKe7D3hNQtQaHeHt5L8LO1CB8GF8jlYVzOv5cBWDNqiJ+oPEWrWei3i3dkZtHY/bUtd0R/uOQ==", + "dev": true, + "requires": { + "nanoid": "*" + } + }, "@types/node": { "version": "12.19.4", "resolved": "https://registry.npmjs.org/@types/node/-/node-12.19.4.tgz", @@ -19759,6 +19791,12 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, + "nanoid": { + "version": "3.1.25", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.25.tgz", + "integrity": "sha512-rdwtIXaXCLFAQbnfqDRnI6jaRHp9fTcYBjtFKE8eezcZ7LuLjhUaQGNeMXf1HmRoCH32CLz6XwX0TtxEOS/A3Q==", + "dev": true + }, "p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -19910,10 +19948,9 @@ "optional": true }, "nanoid": { - "version": "3.1.25", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.25.tgz", - "integrity": "sha512-rdwtIXaXCLFAQbnfqDRnI6jaRHp9fTcYBjtFKE8eezcZ7LuLjhUaQGNeMXf1HmRoCH32CLz6XwX0TtxEOS/A3Q==", - "dev": true + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.2.0.tgz", + "integrity": "sha512-fmsZYa9lpn69Ad5eDn7FMcnnSR+8R34W9qJEijxYhTbfOWzr22n1QxCMzXLK+ODyW2973V3Fux959iQoUxzUIA==" }, "nanomatch": { "version": "1.2.13", diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 9443942b72b..feff4f58cce 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -1021,6 +1021,7 @@ "glob-promise": "^3.4.0", "js-yaml": "^3.14.0", "minimist": "~1.2.5", + "nanoid": "^3.2.0", "node-fetch": "~2.6.7", "path-browserify": "^1.0.1", "react": "^17.0.2", @@ -1056,6 +1057,7 @@ "@types/js-yaml": "^3.12.5", "@types/jszip": "~3.1.6", "@types/mocha": "^9.0.0", + "@types/nanoid": "^3.0.0", "@types/node": "^12.14.1", "@types/node-fetch": "~2.5.2", "@types/proxyquire": "~1.3.28", diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts index 05773662274..7de1d021a38 100644 --- a/extensions/ql-vscode/src/extension.ts +++ b/extensions/ql-vscode/src/extension.ts @@ -438,6 +438,7 @@ async function activateWithInstalledDistribution( const qhm = new QueryHistoryManager( qs, + dbm, ctx.extensionPath, queryHistoryConfigurationListener, showResults, diff --git a/extensions/ql-vscode/src/interface.ts b/extensions/ql-vscode/src/interface.ts index 4766855e50e..144ad603403 100644 --- a/extensions/ql-vscode/src/interface.ts +++ b/extensions/ql-vscode/src/interface.ts @@ -316,7 +316,7 @@ export class InterfaceManager extends DisposableObject { // sortedResultsInfo doesn't have an entry for the current // result set. Use this to determine whether or not we use // the sorted bqrs file. - this._displayedQuery?.completedQuery.sortedResultsInfo.has(msg.selectedTable) || false + !!this._displayedQuery?.completedQuery.sortedResultsInfo[msg.selectedTable] ); } break; @@ -372,8 +372,8 @@ export class InterfaceManager extends DisposableObject { ); const sortedResultsMap: SortedResultsMap = {}; - fullQuery.completedQuery.sortedResultsInfo.forEach( - (v, k) => + Object.entries(fullQuery.completedQuery.sortedResultsInfo).forEach( + ([k, v]) => (sortedResultsMap[k] = this.convertPathPropertiesToWebviewUris(v)) ); @@ -458,7 +458,7 @@ export class InterfaceManager extends DisposableObject { shouldKeepOldResultsWhileRendering, metadata: fullQuery.completedQuery.query.metadata, queryName: fullQuery.label, - queryPath: fullQuery.completedQuery.query.program.queryPath + queryPath: fullQuery.initialInfo.queryPath }); } @@ -491,7 +491,7 @@ export class InterfaceManager extends DisposableObject { pageSize: PAGE_SIZE.getValue(), numPages: numInterpretedPages(this._interpretation), queryName: this._displayedQuery.label, - queryPath: this._displayedQuery.completedQuery.query.program.queryPath + queryPath: this._displayedQuery.initialInfo.queryPath }); } @@ -523,8 +523,8 @@ export class InterfaceManager extends DisposableObject { } const sortedResultsMap: SortedResultsMap = {}; - results.completedQuery.sortedResultsInfo.forEach( - (v, k) => + Object.entries(results.completedQuery.sortedResultsInfo).forEach( + ([k, v]) => (sortedResultsMap[k] = this.convertPathPropertiesToWebviewUris(v)) ); @@ -576,7 +576,7 @@ export class InterfaceManager extends DisposableObject { shouldKeepOldResultsWhileRendering: false, metadata: results.completedQuery.query.metadata, queryName: results.label, - queryPath: results.completedQuery.query.program.queryPath + queryPath: results.initialInfo.queryPath }); } @@ -649,14 +649,18 @@ export class InterfaceManager extends DisposableObject { sortState: InterpretedResultsSortState | undefined ): Promise { if ( - (await query.canHaveInterpretedResults()) && + query.canHaveInterpretedResults() && query.quickEvalPosition === undefined // never do results interpretation if quickEval ) { try { - const sourceLocationPrefix = await query.dbItem.getSourceLocationPrefix( + const dbItem = this.databaseManager.findDatabaseItem(Uri.file(query.dbItemPath)); + if (!dbItem) { + throw new Error(`Could not find database item for ${query.dbItemPath}`); + } + const sourceLocationPrefix = await dbItem.getSourceLocationPrefix( this.cliServer ); - const sourceArchiveUri = query.dbItem.sourceArchive; + const sourceArchiveUri = dbItem.sourceArchive; const sourceInfo = sourceArchiveUri === undefined ? undefined diff --git a/extensions/ql-vscode/src/query-history.ts b/extensions/ql-vscode/src/query-history.ts index 2e5928414ee..cd9e9983057 100644 --- a/extensions/ql-vscode/src/query-history.ts +++ b/extensions/ql-vscode/src/query-history.ts @@ -28,6 +28,7 @@ import { DisposableObject } from './pure/disposable-object'; import { commandRunner } from './commandRunner'; import { assertNever } from './pure/helpers-pure'; import { FullCompletedQueryInfo, FullQueryInfo, QueryStatus } from './query-results'; +import { DatabaseManager } from './databases'; /** * query-history.ts @@ -246,6 +247,7 @@ export class QueryHistoryManager extends DisposableObject { constructor( private qs: QueryServerClient, + private dbm: DatabaseManager, extensionPath: string, queryHistoryConfigListener: QueryHistoryConfig, private selectedCallback: (item: FullCompletedQueryInfo) => Promise, @@ -599,14 +601,12 @@ export class QueryHistoryManager extends DisposableObject { throw new Error(NO_QUERY_SELECTED); } - const rawQueryName = singleItem.getQueryName(); - const queryName = rawQueryName.endsWith('.ql') ? rawQueryName : rawQueryName + '.ql'; const params = new URLSearchParams({ isQuickEval: String(!!singleItem.initialInfo.quickEvalPosition), queryText: encodeURIComponent(await this.getQueryText(singleItem)), }); const uri = Uri.parse( - `codeql:${singleItem.initialInfo.id}-${queryName}?${params.toString()}`, true + `codeql:${singleItem.initialInfo.id}?${params.toString()}`, true ); const doc = await workspace.openTextDocument(uri); await window.showTextDocument(doc, { preview: false }); @@ -620,7 +620,7 @@ export class QueryHistoryManager extends DisposableObject { return; } const query = singleItem.completedQuery.query; - const hasInterpretedResults = await query.canHaveInterpretedResults(); + const hasInterpretedResults = query.canHaveInterpretedResults(); if (hasInterpretedResults) { await this.tryOpenExternalFile( query.resultsPaths.interpretedResultsPath @@ -664,7 +664,7 @@ export class QueryHistoryManager extends DisposableObject { } await this.tryOpenExternalFile( - await singleItem.completedQuery.query.ensureCsvProduced(this.qs) + await singleItem.completedQuery.query.ensureCsvProduced(this.qs, this.dbm) ); } diff --git a/extensions/ql-vscode/src/query-results.ts b/extensions/ql-vscode/src/query-results.ts index d3ad46c69b2..d5bb64873a7 100644 --- a/extensions/ql-vscode/src/query-results.ts +++ b/extensions/ql-vscode/src/query-results.ts @@ -1,6 +1,6 @@ import { CancellationTokenSource, env } from 'vscode'; -import { QueryWithResults, tmpDir, QueryEvaluationInfo } from './run-queries'; +import { QueryWithResults, QueryEvaluationInfo } from './run-queries'; import * as messages from './pure/messages'; import * as cli from './cli'; import * as sarif from 'sarif'; @@ -15,6 +15,7 @@ import { } from './pure/interface-types'; import { QueryHistoryConfig } from './config'; import { DatabaseInfo } from './pure/interface-types'; +import { showAndLogErrorMessage } from './helpers'; /** * A description of the information about a query @@ -29,7 +30,7 @@ export interface InitialQueryInfo { readonly queryPath: string; readonly databaseInfo: DatabaseInfo readonly start: Date; - readonly id: number; // an incrementing number for each query + readonly id: string; // unique id for this query. } export enum QueryStatus { @@ -43,12 +44,16 @@ export class CompletedQueryInfo implements QueryWithResults { readonly result: messages.EvaluationResult; readonly logFileLocation?: string; resultCount: number; + + /** + * This dispose method is called when the query is removed from the history view. + */ dispose: () => void; /** * Map from result set name to SortedResultSetInfo. */ - sortedResultsInfo: Map; + sortedResultsInfo: Record; /** * How we're currently sorting alerts. This is not mere interface @@ -59,15 +64,23 @@ export class CompletedQueryInfo implements QueryWithResults { */ interpretedResultsSortState: InterpretedResultsSortState | undefined; + /** + * Note that in the {@link FullQueryInfo.slurp} method, we create a CompletedQueryInfo instance + * by explicitly setting the prototype in order to avoid calling this constructor. + */ constructor( evaluation: QueryWithResults, ) { this.query = evaluation.query; this.result = evaluation.result; this.logFileLocation = evaluation.logFileLocation; + + // Use the dispose method from the evaluation. + // The dispose will clean up any additional log locations that this + // query may have created. this.dispose = evaluation.dispose; - this.sortedResultsInfo = new Map(); + this.sortedResultsInfo = {}; this.resultCount = 0; } @@ -95,7 +108,7 @@ export class CompletedQueryInfo implements QueryWithResults { if (!useSorted) { return this.query.resultsPaths.resultsPath; } - return this.sortedResultsInfo.get(selectedTable)?.resultsPath + return this.sortedResultsInfo[selectedTable]?.resultsPath || this.query.resultsPaths.resultsPath; } @@ -109,12 +122,12 @@ export class CompletedQueryInfo implements QueryWithResults { sortState?: RawResultsSortState ): Promise { if (sortState === undefined) { - this.sortedResultsInfo.delete(resultSetName); + delete this.sortedResultsInfo[resultSetName]; return; } const sortedResultSetInfo: SortedResultSetInfo = { - resultsPath: path.join(tmpDir.name, `sortedResults${this.query.queryID}-${resultSetName}.bqrs`), + resultsPath: this.query.getSortedResultSetPath(resultSetName), sortState }; @@ -125,7 +138,7 @@ export class CompletedQueryInfo implements QueryWithResults { [sortState.columnIndex], [sortState.sortDirection] ); - this.sortedResultsInfo.set(resultSetName, sortedResultSetInfo); + this.sortedResultsInfo[resultSetName] = sortedResultSetInfo; } async updateInterpretedSortState(sortState?: InterpretedResultsSortState): Promise { @@ -174,19 +187,79 @@ export type FullCompletedQueryInfo = FullQueryInfo & { }; export class FullQueryInfo { + + static async slurp(fsPath: string, config: QueryHistoryConfig): Promise { + try { + const data = await fs.readFile(fsPath, 'utf8'); + const queries = JSON.parse(data); + return queries.map((q: FullQueryInfo) => { + + // Need to explicitly set prototype since reading in from JSON will not + // do this automatically. Note that we can't call the constructor here since + // the constructor invokes extra logic that we don't want to do. + Object.setPrototypeOf(q, FullQueryInfo.prototype); + + // The config object is a global, se we need to set it explicitly + // and ensure it is not serialized to JSON. + q.setConfig(config); + + // Date instances are serialized as strings. Need to + // convert them back to Date instances. + (q.initialInfo as any).start = new Date(q.initialInfo.start); + if (q.completedQuery) { + // Again, need to explicitly set prototypes. + Object.setPrototypeOf(q.completedQuery, CompletedQueryInfo.prototype); + Object.setPrototypeOf(q.completedQuery.query, QueryEvaluationInfo.prototype); + // slurped queries do not need to be disposed + q.completedQuery.dispose = () => { /**/ }; + } + return q; + }); + } catch (e) { + void showAndLogErrorMessage('Error loading query history.', { + fullMessage: ['Error loading query history.', e.stack].join('\n'), + }); + return []; + } + } + + /** + * Save the query history to disk. It is not necessary that the parent directory + * exists, but if it does, it must be writable. An existing file will be overwritten. + * + * Any errors will be rethrown. + * + * @param queries the list of queries to save. + * @param fsPath the path to save the queries to. + */ + static async splat(queries: FullQueryInfo[], fsPath: string): Promise { + try { + const data = JSON.stringify(queries, null, 2); + await fs.mkdirp(path.dirname(fsPath)); + await fs.writeFile(fsPath, data); + } catch (e) { + throw new Error(`Error saving query history to ${fsPath}: ${e.message}`); + } + } + public failureReason: string | undefined; public completedQuery: CompletedQueryInfo | undefined; + private config: QueryHistoryConfig | undefined; + /** + * Note that in the {@link FullQueryInfo.slurp} method, we create a FullQueryInfo instance + * by explicitly setting the prototype in order to avoid calling this constructor. + */ constructor( public readonly initialInfo: InitialQueryInfo, - private readonly config: QueryHistoryConfig, - private readonly source: CancellationTokenSource + config: QueryHistoryConfig, + private readonly source?: CancellationTokenSource ) { - /**/ + this.setConfig(config); } cancel() { - this.source.cancel(); + this.source?.cancel(); } get startTime() { @@ -214,7 +287,9 @@ export class FullQueryInfo { * Returns a label for this query that includes interpolated values. */ get label(): string { - return this.interpolate(this.initialInfo.userSpecifiedLabel ?? this.config.format ?? ''); + return this.interpolate( + this.initialInfo.userSpecifiedLabel ?? this.config?.format ?? '' + ); } /** @@ -286,4 +361,21 @@ export class FullQueryInfo { return QueryStatus.Failed; } } + + /** + * The `config` property must not be serialized since it contains a listerner + * for global configuration changes. Instead, It should be set when the query + * is deserialized. + * + * @param config the global query history config object + */ + private setConfig(config: QueryHistoryConfig) { + // avoid serializing config property + Object.defineProperty(this, 'config', { + enumerable: false, + writable: false, + configurable: true, + value: config + }); + } } diff --git a/extensions/ql-vscode/src/run-queries.ts b/extensions/ql-vscode/src/run-queries.ts index 185fedd6da6..5e781a1fed4 100644 --- a/extensions/ql-vscode/src/run-queries.ts +++ b/extensions/ql-vscode/src/run-queries.ts @@ -2,6 +2,7 @@ import * as crypto from 'crypto'; import * as fs from 'fs-extra'; import * as path from 'path'; import * as tmp from 'tmp-promise'; +import { nanoid } from 'nanoid'; import { CancellationToken, ConfigurationTarget, @@ -16,10 +17,10 @@ import { ErrorCodes, ResponseError } from 'vscode-languageclient'; import * as cli from './cli'; import * as config from './config'; -import { DatabaseItem } from './databases'; +import { DatabaseItem, DatabaseManager } from './databases'; import { getOnDiskWorkspaceFolders, showAndLogErrorMessage, tryGetQueryMetadata } from './helpers'; import { ProgressCallback, UserCancellationException } from './commandRunner'; -import { DatabaseInfo, QueryMetadata, ResultsPaths } from './pure/interface-types'; +import { DatabaseInfo, QueryMetadata } from './pure/interface-types'; import { logger } from './logging'; import * as messages from './pure/messages'; import { InitialQueryInfo } from './query-results'; @@ -37,7 +38,6 @@ import { DecodedBqrsChunk } from './pure/bqrs-cli-types'; * Compiling and running QL queries. */ -// XXX: Tmp directory should be configuarble. export const tmpDir = tmp.dirSync({ prefix: 'queries_', keep: false, unsafeCleanup: true }); export const upgradesTmpDir = tmp.dirSync({ dir: tmpDir.name, prefix: 'upgrades_', keep: false, unsafeCleanup: true }); export const tmpDirDisposal = { @@ -47,6 +47,9 @@ export const tmpDirDisposal = { } }; +// exported for testing +export const queriesDir = path.join(tmpDir.name, 'queries'); + /** * A collection of evaluation-time information about a query, * including the query itself, and where we have decided to put @@ -54,41 +57,59 @@ export const tmpDirDisposal = { * output and results. */ export class QueryEvaluationInfo { - readonly compiledQueryPath: string; - readonly dilPath: string; - readonly csvPath: string; - readonly resultsPaths: ResultsPaths; - readonly dataset: Uri; // guarantee the existence of a well-defined dataset dir at this point + readonly querySaveDir: string; + /** + * Note that in the {@link FullQueryInfo.slurp} method, we create a QueryEvaluationInfo instance + * by explicitly setting the prototype in order to avoid calling this constructor. + */ constructor( - public readonly queryID: number, - public readonly program: messages.QlProgram, - public readonly dbItem: DatabaseItem, + public readonly id: string, + public readonly dbItemPath: string, + private readonly databaseHasMetadataFile: boolean, public readonly queryDbscheme: string, // the dbscheme file the query expects, based on library path resolution public readonly quickEvalPosition?: messages.Position, public readonly metadata?: QueryMetadata, public readonly templates?: messages.TemplateDefinitions ) { - this.compiledQueryPath = path.join(tmpDir.name, `compiledQuery${this.queryID}.qlo`); - this.dilPath = path.join(tmpDir.name, `results${this.queryID}.dil`); - this.csvPath = path.join(tmpDir.name, `results${this.queryID}.csv`); - this.resultsPaths = { - resultsPath: path.join(tmpDir.name, `results${this.queryID}.bqrs`), - interpretedResultsPath: path.join(tmpDir.name, `interpretedResults${this.queryID}.sarif`) + this.querySaveDir = path.join(queriesDir, this.id); + } + + get dilPath() { + return path.join(this.querySaveDir, 'results.dil'); + } + + get csvPath() { + return path.join(this.querySaveDir, 'results.csv'); + } + + get compiledQueryPath() { + return path.join(this.querySaveDir, 'compiledQuery.qlo'); + } + + get resultsPaths() { + return { + resultsPath: path.join(this.querySaveDir, 'results.bqrs'), + interpretedResultsPath: path.join(this.querySaveDir, 'interpretedResults.sarif'), }; - if (dbItem.contents === undefined) { - throw new Error('Can\'t run query on invalid database.'); - } - this.dataset = dbItem.contents.datasetUri; + } + + getSortedResultSetPath(resultSetName: string) { + return path.join(this.querySaveDir, `sortedResults-${resultSetName}.bqrs`); } async run( qs: qsClient.QueryServerClient, upgradeQlo: string | undefined, availableMlModels: cli.MlModelInfo[], + dbItem: DatabaseItem, progress: ProgressCallback, token: CancellationToken, ): Promise { + if (!dbItem.contents || dbItem.error) { + throw new Error('Can\'t run query on invalid database.'); + } + let result: messages.EvaluationResult | null = null; const callbackId = qs.registerCallback(res => { result = res; }); @@ -106,7 +127,7 @@ export class QueryEvaluationInfo { timeoutSecs: qs.config.timeoutSecs, }; const dataset: messages.Dataset = { - dbDir: this.dataset.fsPath, + dbDir: dbItem.contents.datasetUri.fsPath, workingSet: 'default' }; const params: messages.EvaluateQueriesParams = { @@ -132,6 +153,7 @@ export class QueryEvaluationInfo { async compile( qs: qsClient.QueryServerClient, + program: messages.QlProgram, progress: ProgressCallback, token: CancellationToken, ): Promise { @@ -154,7 +176,7 @@ export class QueryEvaluationInfo { extraOptions: { timeoutSecs: qs.config.timeoutSecs }, - queryToCheck: this.program, + queryToCheck: program, resultPath: this.compiledQueryPath, target, }; @@ -169,20 +191,22 @@ export class QueryEvaluationInfo { /** * Holds if this query can in principle produce interpreted results. */ - async canHaveInterpretedResults(): Promise { - const hasMetadataFile = await this.dbItem.hasMetadataFile(); - if (!hasMetadataFile) { + canHaveInterpretedResults(): boolean { + if (!this.databaseHasMetadataFile) { void logger.log('Cannot produce interpreted results since the database does not have a .dbinfo or codeql-database.yml file.'); + return false; } const hasKind = !!this.metadata?.kind; if (!hasKind) { void logger.log('Cannot produce interpreted results since the query does not have @kind metadata.'); + return false; } + // table is the default query kind. It does not produce interpreted results. + // any query kind that is not table can, in principle, produce interpreted results. const isTable = hasKind && this.metadata?.kind === 'table'; - - return hasMetadataFile && hasKind && !isTable; + return !isTable; } /** @@ -244,16 +268,21 @@ export class QueryEvaluationInfo { out.end(); } - async ensureCsvProduced(qs: qsClient.QueryServerClient): Promise { + async ensureCsvProduced(qs: qsClient.QueryServerClient, dbm: DatabaseManager): Promise { if (await this.hasCsv()) { return this.csvPath; } + const dbItem = dbm.findDatabaseItem(Uri.file(this.dbItemPath)); + if (!dbItem) { + throw new Error(`Cannot produce CSV results because database is missing. ${this.dbItemPath}`); + } + let sourceInfo; - if (this.dbItem.sourceArchive !== undefined) { + if (dbItem.sourceArchive !== undefined) { sourceInfo = { - sourceArchive: this.dbItem.sourceArchive.fsPath, - sourceLocationPrefix: await this.dbItem.getSourceLocationPrefix( + sourceArchive: dbItem.sourceArchive.fsPath, + sourceLocationPrefix: await dbItem.getSourceLocationPrefix( qs.cliServer ), }; @@ -264,7 +293,6 @@ export class QueryEvaluationInfo { } } - export interface QueryWithResults { readonly query: QueryEvaluationInfo; readonly result: messages.EvaluationResult; @@ -352,32 +380,32 @@ async function checkDbschemeCompatibility( cliServer: cli.CodeQLCliServer, qs: qsClient.QueryServerClient, query: QueryEvaluationInfo, + qlProgram: messages.QlProgram, + dbItem: DatabaseItem, progress: ProgressCallback, token: CancellationToken, ): Promise { const searchPath = getOnDiskWorkspaceFolders(); - if (query.dbItem.contents !== undefined && query.dbItem.contents.dbSchemeUri !== undefined) { - const { finalDbscheme } = await cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath, false); + if (dbItem.contents?.dbSchemeUri !== undefined) { + const { finalDbscheme } = await cliServer.resolveUpgrades(dbItem.contents.dbSchemeUri.fsPath, searchPath, false); const hash = async function(filename: string): Promise { return crypto.createHash('sha256').update(await fs.readFile(filename)).digest('hex'); }; // At this point, we have learned about three dbschemes: - // query.program.dbschemePath is the dbscheme of the actual - // database we're querying. - const dbschemeOfDb = await hash(query.program.dbschemePath); + // the dbscheme of the actual database we're querying. + const dbschemeOfDb = await hash(dbItem.contents.dbSchemeUri.fsPath); - // query.queryDbScheme is the dbscheme of the query we're - // running, including the library we've resolved it to use. + // the dbscheme of the query we're running, including the library we've resolved it to use. const dbschemeOfLib = await hash(query.queryDbscheme); - // info.finalDbscheme is which database we're able to upgrade to + // the database we're able to upgrade to const upgradableTo = await hash(finalDbscheme); if (upgradableTo != dbschemeOfLib) { - reportNoUpgradePath(query); + reportNoUpgradePath(qlProgram, query); } if (upgradableTo == dbschemeOfLib && @@ -385,7 +413,7 @@ async function checkDbschemeCompatibility( // Try to upgrade the database await upgradeDatabaseExplicit( qs, - query.dbItem, + dbItem, progress, token ); @@ -393,8 +421,8 @@ async function checkDbschemeCompatibility( } } -function reportNoUpgradePath(query: QueryEvaluationInfo) { - throw new Error(`Query ${query.program.queryPath} expects database scheme ${query.queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace.\n\nPlease try using a newer version of the query libraries.`); +function reportNoUpgradePath(qlProgram: messages.QlProgram, query: QueryEvaluationInfo): void { + throw new Error(`Query ${qlProgram.queryPath} expects database scheme ${query.queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace.\n\nPlease try using a newer version of the query libraries.`); } /** @@ -404,26 +432,28 @@ async function compileNonDestructiveUpgrade( qs: qsClient.QueryServerClient, upgradeTemp: tmp.DirectoryResult, query: QueryEvaluationInfo, + qlProgram: messages.QlProgram, + dbItem: DatabaseItem, progress: ProgressCallback, token: CancellationToken, ): Promise { const searchPath = getOnDiskWorkspaceFolders(); - if (!query.dbItem?.contents?.dbSchemeUri) { + if (!dbItem?.contents?.dbSchemeUri) { throw new Error('Database is invalid, and cannot be upgraded.'); } - const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath, true, query.queryDbscheme); + const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades(dbItem.contents.dbSchemeUri.fsPath, searchPath, true, query.queryDbscheme); if (!matchesTarget) { - reportNoUpgradePath(query); + reportNoUpgradePath(qlProgram, query); } - const result = await compileDatabaseUpgradeSequence(qs, query.dbItem, scripts, upgradeTemp, progress, token); + const result = await compileDatabaseUpgradeSequence(qs, dbItem, scripts, upgradeTemp, progress, token); if (result.compiledUpgrade === undefined) { const error = result.error || '[no error message available]'; throw new Error(error); } // We can upgrade to the actual target - query.program.dbschemePath = query.queryDbscheme; + qlProgram.dbschemePath = query.queryDbscheme; // We are new enough that we will always support single file upgrades. return result.compiledUpgrade; @@ -508,14 +538,13 @@ export async function determineSelectedQuery(selectedResourceUri: Uri | undefine if (queryUri.scheme !== 'file') { throw new Error('Can only run queries that are on disk.'); } - const queryPath = queryUri.fsPath || ''; + const queryPath = queryUri.fsPath; if (quickEval) { if (!(queryPath.endsWith('.ql') || queryPath.endsWith('.qll'))) { throw new Error('The selected resource is not a CodeQL file; It should have the extension ".ql" or ".qll".'); } - } - else { + } else { if (!(queryPath.endsWith('.ql'))) { throw new Error('The selected resource is not a CodeQL query file; It should have the extension ".ql".'); } @@ -557,14 +586,14 @@ export async function determineSelectedQuery(selectedResourceUri: Uri | undefine export async function compileAndRunQueryAgainstDatabase( cliServer: cli.CodeQLCliServer, qs: qsClient.QueryServerClient, - db: DatabaseItem, + dbItem: DatabaseItem, initialInfo: InitialQueryInfo, progress: ProgressCallback, token: CancellationToken, templates?: messages.TemplateDefinitions, ): Promise { - if (!db.contents || !db.contents.dbSchemeUri) { - throw new Error(`Database ${db.databaseUri} does not have a CodeQL database scheme.`); + if (!dbItem.contents || !dbItem.contents.dbSchemeUri) { + throw new Error(`Database ${dbItem.databaseUri} does not have a CodeQL database scheme.`); } // Get the workspace folder paths. @@ -581,10 +610,10 @@ export async function compileAndRunQueryAgainstDatabase( // won't trigger this check) // This test will produce confusing results if we ever change the name of the database schema files. const querySchemaName = path.basename(packConfig.dbscheme); - const dbSchemaName = path.basename(db.contents.dbSchemeUri.fsPath); + const dbSchemaName = path.basename(dbItem.contents.dbSchemeUri.fsPath); if (querySchemaName != dbSchemaName) { void logger.log(`Query schema was ${querySchemaName}, but database schema was ${dbSchemaName}.`); - throw new Error(`The query ${path.basename(initialInfo.queryPath)} cannot be run against the selected database (${db.name}): their target languages are different. Please select a different database and try again.`); + throw new Error(`The query ${path.basename(initialInfo.queryPath)} cannot be run against the selected database (${dbItem.name}): their target languages are different. Please select a different database and try again.`); } const qlProgram: messages.QlProgram = { @@ -595,7 +624,7 @@ export async function compileAndRunQueryAgainstDatabase( // Since we are compiling and running a query against a database, // we use the database's DB scheme here instead of the DB scheme // from the current document's project. - dbschemePath: db.contents.dbSchemeUri.fsPath, + dbschemePath: dbItem.contents.dbSchemeUri.fsPath, queryPath: initialInfo.queryPath }; @@ -620,19 +649,28 @@ export async function compileAndRunQueryAgainstDatabase( } } - const query = new QueryEvaluationInfo(initialInfo.id, qlProgram, db, packConfig.dbscheme, initialInfo.quickEvalPosition, metadata, templates); + const hasMetadataFile = (await dbItem.hasMetadataFile()); + const query = new QueryEvaluationInfo( + initialInfo.id, + dbItem.databaseUri.fsPath, + hasMetadataFile, + packConfig.dbscheme, + initialInfo.quickEvalPosition, + metadata, + templates + ); const upgradeDir = await tmp.dir({ dir: upgradesTmpDir.name, unsafeCleanup: true }); try { let upgradeQlo; if (await hasNondestructiveUpgradeCapabilities(qs)) { - upgradeQlo = await compileNonDestructiveUpgrade(qs, upgradeDir, query, progress, token); + upgradeQlo = await compileNonDestructiveUpgrade(qs, upgradeDir, query, qlProgram, dbItem, progress, token); } else { - await checkDbschemeCompatibility(cliServer, qs, query, progress, token); + await checkDbschemeCompatibility(cliServer, qs, query, qlProgram, dbItem, progress, token); } let errors; try { - errors = await query.compile(qs, progress, token); + errors = await query.compile(qs, qlProgram, progress, token); } catch (e) { if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) { return createSyntheticResult(query, 'Query cancelled', messages.QueryResultType.CANCELLATION); @@ -642,7 +680,7 @@ export async function compileAndRunQueryAgainstDatabase( } if (errors.length === 0) { - const result = await query.run(qs, upgradeQlo, availableMlModels, progress, token); + const result = await query.run(qs, upgradeQlo, availableMlModels, dbItem, progress, token); if (result.resultType !== messages.QueryResultType.SUCCESS) { const message = result.message || 'Failed to run query'; void logger.log(message); @@ -661,7 +699,7 @@ export async function compileAndRunQueryAgainstDatabase( // so we include a general description of the problem, // and direct the user to the output window for the detailed compilation messages. // However we don't show quick eval errors there so we need to display them anyway. - void qs.logger.log(`Failed to compile query ${query.program.queryPath} against database scheme ${query.program.dbschemePath}:`); + void qs.logger.log(`Failed to compile query ${initialInfo.queryPath} against database scheme ${qlProgram.dbschemePath}:`); const formattedMessages: string[] = []; @@ -691,7 +729,6 @@ export async function compileAndRunQueryAgainstDatabase( } } -let queryId = 0; export async function createInitialQueryInfo( selectedQueryUri: Uri | undefined, databaseInfo: DatabaseInfo, @@ -706,7 +743,7 @@ export async function createInitialQueryInfo( isQuickEval, isQuickQuery: isQuickQueryPath(queryPath), databaseInfo, - id: queryId++, + id: `${path.basename(queryPath)}-${nanoid()}`, start: new Date(), ... (isQuickEval ? { queryText: quickEvalText!, // if this query is quick eval, it must have quick eval text diff --git a/extensions/ql-vscode/src/upgrades.ts b/extensions/ql-vscode/src/upgrades.ts index f7818e3750f..8c5543bca1a 100644 --- a/extensions/ql-vscode/src/upgrades.ts +++ b/extensions/ql-vscode/src/upgrades.ts @@ -35,13 +35,13 @@ export async function hasNondestructiveUpgradeCapabilities(qs: qsClient.QuerySer */ export async function compileDatabaseUpgradeSequence( qs: qsClient.QueryServerClient, - db: DatabaseItem, + dbItem: DatabaseItem, resolvedSequence: string[], currentUpgradeTmp: tmp.DirectoryResult, progress: ProgressCallback, token: vscode.CancellationToken ): Promise { - if (db.contents === undefined || db.contents.dbSchemeUri === undefined) { + if (dbItem.contents === undefined || dbItem.contents.dbSchemeUri === undefined) { throw new Error('Database is invalid, and cannot be upgraded.'); } if (!await hasNondestructiveUpgradeCapabilities(qs)) { @@ -56,14 +56,14 @@ export async function compileDatabaseUpgradeSequence( async function compileDatabaseUpgrade( qs: qsClient.QueryServerClient, - db: DatabaseItem, + dbItem: DatabaseItem, targetDbScheme: string, resolvedSequence: string[], currentUpgradeTmp: tmp.DirectoryResult, progress: ProgressCallback, token: vscode.CancellationToken ): Promise { - if (!db.contents?.dbSchemeUri) { + if (!dbItem.contents?.dbSchemeUri) { throw new Error('Database is invalid, and cannot be upgraded.'); } // We have the upgrades we want but compileUpgrade @@ -78,7 +78,7 @@ async function compileDatabaseUpgrade( }); return qs.sendRequest(messages.compileUpgrade, { upgrade: { - fromDbscheme: db.contents.dbSchemeUri.fsPath, + fromDbscheme: dbItem.contents.dbSchemeUri.fsPath, toDbscheme: targetDbScheme, additionalUpgrades: Array.from(uniqueParentDirs) }, @@ -159,18 +159,18 @@ function getUpgradeDescriptions(compiled: messages.CompiledUpgrades): messages.U */ export async function upgradeDatabaseExplicit( qs: qsClient.QueryServerClient, - db: DatabaseItem, + dbItem: DatabaseItem, progress: ProgressCallback, token: vscode.CancellationToken, ): Promise { const searchPath: string[] = getOnDiskWorkspaceFolders(); - if (!db?.contents?.dbSchemeUri) { + if (!dbItem?.contents?.dbSchemeUri) { throw new Error('Database is invalid, and cannot be upgraded.'); } const upgradeInfo = await qs.cliServer.resolveUpgrades( - db.contents.dbSchemeUri.fsPath, + dbItem.contents.dbSchemeUri.fsPath, searchPath, false ); @@ -184,7 +184,7 @@ export async function upgradeDatabaseExplicit( try { let compileUpgradeResult: messages.CompileUpgradeResult; try { - compileUpgradeResult = await compileDatabaseUpgrade(qs, db, finalDbscheme, scripts, currentUpgradeTmp, progress, token); + compileUpgradeResult = await compileDatabaseUpgrade(qs, dbItem, finalDbscheme, scripts, currentUpgradeTmp, progress, token); } catch (e) { void showAndLogErrorMessage(`Compilation of database upgrades failed: ${e}`); @@ -200,13 +200,13 @@ export async function upgradeDatabaseExplicit( return; } - await checkAndConfirmDatabaseUpgrade(compileUpgradeResult.compiledUpgrades, db, qs.cliServer.quiet); + await checkAndConfirmDatabaseUpgrade(compileUpgradeResult.compiledUpgrades, dbItem, qs.cliServer.quiet); try { void qs.logger.log('Running the following database upgrade:'); getUpgradeDescriptions(compileUpgradeResult.compiledUpgrades).map(s => s.description).join('\n'); - return await runDatabaseUpgrade(qs, db, compileUpgradeResult.compiledUpgrades, progress, token); + return await runDatabaseUpgrade(qs, dbItem, compileUpgradeResult.compiledUpgrades, progress, token); } catch (e) { void showAndLogErrorMessage(`Database upgrade failed: ${e}`); diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/query-history.test.ts b/extensions/ql-vscode/src/vscode-tests/no-workspace/query-history.test.ts index c673fb56dfb..0f7c20eb91a 100644 --- a/extensions/ql-vscode/src/vscode-tests/no-workspace/query-history.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/query-history.test.ts @@ -11,6 +11,7 @@ import { QueryHistoryConfigListener } from '../../config'; import * as messages from '../../pure/messages'; import { QueryServerClient } from '../../queryserver-client'; import { FullQueryInfo, InitialQueryInfo } from '../../query-results'; +import { DatabaseManager } from '../../databases'; chai.use(chaiAsPromised); const expect = chai.expect; @@ -211,8 +212,11 @@ describe('query-history', () => { }); }); - it('should remove an item and not select a new one', async function() { + it('should remove an item and not select a new one', async () => { queryHistoryManager = await createMockQueryHistory(allHistory); + // initialize the selection + await queryHistoryManager.treeView.reveal(allHistory[0], { select: true }); + // deleting the first item when a different item is selected // will not change the selection const toDelete = allHistory[1]; @@ -220,13 +224,18 @@ describe('query-history', () => { // select the item we want await queryHistoryManager.treeView.reveal(selected, { select: true }); + + // should be selected + expect(queryHistoryManager.treeDataProvider.getCurrent()).to.deep.eq(selected); + + // remove an item await queryHistoryManager.handleRemoveHistoryItem(toDelete, [toDelete]); expect(toDelete.completedQuery!.dispose).to.have.been.calledOnce; expect(queryHistoryManager.treeDataProvider.getCurrent()).to.deep.eq(selected); expect(queryHistoryManager.treeDataProvider.allHistory).not.to.contain(toDelete); - // the current item should have been re-selected + // the same item should be selected expect(selectedCallback).to.have.been.calledOnceWith(selected); }); @@ -545,6 +554,7 @@ describe('query-history', () => { async function createMockQueryHistory(allHistory: FullQueryInfo[]) { const qhm = new QueryHistoryManager( {} as QueryServerClient, + {} as DatabaseManager, 'xxx', configListener, selectedCallback, diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/query-results.test.ts b/extensions/ql-vscode/src/vscode-tests/no-workspace/query-results.test.ts index 26fab2de853..85d6819a540 100644 --- a/extensions/ql-vscode/src/vscode-tests/no-workspace/query-results.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/query-results.test.ts @@ -6,13 +6,13 @@ import 'sinon-chai'; import * as sinon from 'sinon'; import * as chaiAsPromised from 'chai-as-promised'; import { FullQueryInfo, InitialQueryInfo, interpretResults } from '../../query-results'; -import { QueryEvaluationInfo, QueryWithResults, tmpDir } from '../../run-queries'; +import { queriesDir, QueryEvaluationInfo, QueryWithResults, tmpDir } from '../../run-queries'; import { QueryHistoryConfig } from '../../config'; import { EvaluationResult, QueryResultType } from '../../pure/messages'; -import { SortDirection, SortedResultSetInfo } from '../../pure/interface-types'; +import { DatabaseInfo, SortDirection, SortedResultSetInfo } from '../../pure/interface-types'; import { CodeQLCliServer, SourceInfo } from '../../cli'; import { env } from 'process'; -import { CancellationTokenSource } from 'vscode'; +import { CancellationTokenSource, Uri } from 'vscode'; chai.use(chaiAsPromised); const expect = chai.expect; @@ -20,12 +20,14 @@ const expect = chai.expect; describe('query-results', () => { let disposeSpy: sinon.SinonSpy; let onDidChangeQueryHistoryConfigurationSpy: sinon.SinonSpy; + let mockConfig: QueryHistoryConfig; let sandbox: sinon.SinonSandbox; beforeEach(() => { sandbox = sinon.createSandbox(); disposeSpy = sandbox.spy(); onDidChangeQueryHistoryConfigurationSpy = sandbox.spy(); + mockConfig = mockQueryHistoryConfig(); }); afterEach(() => { @@ -102,15 +104,17 @@ describe('query-results', () => { it('should get the getResultsPath', () => { const fqi = createMockFullQueryInfo('a', createMockQueryWithResults()); const completedQuery = fqi.completedQuery!; + const expectedResultsPath = path.join(queriesDir, 'some-id/results.bqrs'); + // from results path - expect(completedQuery.getResultsPath('zxa', false)).to.eq('/a/b/c'); + expect(completedQuery.getResultsPath('zxa', false)).to.eq(expectedResultsPath); - completedQuery.sortedResultsInfo.set('zxa', { + completedQuery.sortedResultsInfo['zxa'] = { resultsPath: 'bxa' - } as SortedResultSetInfo); + } as SortedResultSetInfo; // still from results path - expect(completedQuery.getResultsPath('zxa', false)).to.eq('/a/b/c'); + expect(completedQuery.getResultsPath('zxa', false)).to.eq(expectedResultsPath); // from sortedResultsInfo expect(completedQuery.getResultsPath('zxa')).to.eq('bxa'); @@ -141,6 +145,7 @@ describe('query-results', () => { }); it('should updateSortState', async () => { + // setup const fqi = createMockFullQueryInfo('a', createMockQueryWithResults()); const completedQuery = fqi.completedQuery!; @@ -152,24 +157,29 @@ describe('query-results', () => { columnIndex: 1, sortDirection: SortDirection.desc }; - await completedQuery.updateSortState(mockServer, 'result-name', sortState); - const expectedPath = path.join(tmpDir.name, 'sortedResults6789-result-name.bqrs'); + + // test + await completedQuery.updateSortState(mockServer, 'a-result-set-name', sortState); + + // verify + const expectedResultsPath = path.join(queriesDir, 'some-id/results.bqrs'); + const expectedSortedResultsPath = path.join(queriesDir, 'some-id/sortedResults-a-result-set-name.bqrs'); expect(spy).to.have.been.calledWith( - '/a/b/c', - expectedPath, - 'result-name', + expectedResultsPath, + expectedSortedResultsPath, + 'a-result-set-name', [sortState.columnIndex], [sortState.sortDirection], ); - expect(completedQuery.sortedResultsInfo.get('result-name')).to.deep.equal({ - resultsPath: expectedPath, + expect(completedQuery.sortedResultsInfo['a-result-set-name']).to.deep.equal({ + resultsPath: expectedSortedResultsPath, sortState }); - // delete the sort stae - await completedQuery.updateSortState(mockServer, 'result-name'); - expect(completedQuery.sortedResultsInfo.size).to.eq(0); + // delete the sort state + await completedQuery.updateSortState(mockServer, 'a-result-set-name'); + expect(Object.values(completedQuery.sortedResultsInfo).length).to.eq(0); }); }); @@ -237,19 +247,74 @@ describe('query-results', () => { expect(results3).to.deep.eq({ a: 6 }); }); - function createMockQueryWithResults(didRunSuccessfully = true, hasInterpretedResults = true): QueryWithResults { - return { - query: { - hasInterpretedResults: () => Promise.resolve(hasInterpretedResults), - queryID: 6789, - metadata: { - name: 'vwx' - }, - resultsPaths: { - resultsPath: '/a/b/c', - interpretedResultsPath: '/d/e/f' + describe('splat and slurp', () => { + // TODO also add a test for round trip starting from file + it('should splat and slurp query history', async () => { + const infoSuccessRaw = createMockFullQueryInfo('a', createMockQueryWithResults(false, false, '/a/b/c/a', false)); + const infoSuccessInterpreted = createMockFullQueryInfo('b', createMockQueryWithResults(true, true, '/a/b/c/b', false)); + const infoEarlyFailure = createMockFullQueryInfo('c', undefined, true); + const infoLateFailure = createMockFullQueryInfo('d', createMockQueryWithResults(false, false, '/a/b/c/d', false)); + const infoInprogress = createMockFullQueryInfo('e'); + const allHistory = [ + infoSuccessRaw, + infoSuccessInterpreted, + infoEarlyFailure, + infoLateFailure, + infoInprogress + ]; + + const allHistoryPath = path.join(queriesDir, 'all-history.json'); + await FullQueryInfo.splat(allHistory, allHistoryPath); + const allHistoryActual = await FullQueryInfo.slurp(allHistoryPath, mockConfig); + + // the dispose methods will be different. Ignore them. + allHistoryActual.forEach(info => { + if (info.completedQuery) { + const completedQuery = info.completedQuery; + (completedQuery as any).dispose = undefined; + + // these fields should be missing on the slurped value + // but they are undefined on the original value + if (!('logFileLocation' in completedQuery)) { + (completedQuery as any).logFileLocation = undefined; + } + const query = completedQuery.query; + if (!('quickEvalPosition' in query)) { + (query as any).quickEvalPosition = undefined; + } + if (!('templates' in query)) { + (query as any).templates = undefined; + } + } + }); + allHistory.forEach(info => { + if (info.completedQuery) { + (info.completedQuery as any).dispose = undefined; } - } as QueryEvaluationInfo, + }); + + // make the diffs somewhat sane by comparing each element directly + for (let i = 0; i < allHistoryActual.length; i++) { + expect(allHistoryActual[i]).to.deep.eq(allHistory[i]); + } + expect(allHistoryActual.length).to.deep.eq(allHistory.length); + }); + }); + + + function createMockQueryWithResults(didRunSuccessfully = true, hasInterpretedResults = true, dbPath = '/a/b/c', includeSpies = true): QueryWithResults { + const query = new QueryEvaluationInfo('some-id', + Uri.file(dbPath).fsPath, // parse the Uri to make sure it is platform-independent + true, + 'queryDbscheme', + undefined, + { + name: 'vwx' + }, + ); + + const result = { + query, result: { evaluationTime: 12340, resultType: didRunSuccessfully @@ -258,14 +323,27 @@ describe('query-results', () => { } as EvaluationResult, dispose: disposeSpy, }; + + if (includeSpies) { + (query as any).hasInterpretedResults = () => Promise.resolve(hasInterpretedResults); + } + + return result; } function createMockFullQueryInfo(dbName = 'a', queryWitbResults?: QueryWithResults, isFail = false): FullQueryInfo { const fqi = new FullQueryInfo( { - databaseInfo: { name: dbName }, + databaseInfo: { + name: dbName, + databaseUri: Uri.parse(`/a/b/c/${dbName}`).fsPath + } as unknown as DatabaseInfo, start: new Date(), - queryPath: 'path/to/hucairz' + queryPath: 'path/to/hucairz', + queryText: 'some query', + isQuickQuery: false, + isQuickEval: false, + id: `some-id-${dbName}`, } as InitialQueryInfo, mockQueryHistoryConfig(), {} as CancellationTokenSource diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/run-queries.test.ts b/extensions/ql-vscode/src/vscode-tests/no-workspace/run-queries.test.ts index e754370451a..990f520f2dd 100644 --- a/extensions/ql-vscode/src/vscode-tests/no-workspace/run-queries.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/run-queries.test.ts @@ -5,9 +5,9 @@ import 'sinon-chai'; import * as sinon from 'sinon'; import * as chaiAsPromised from 'chai-as-promised'; -import { QueryEvaluationInfo } from '../../run-queries'; -import { QlProgram, Severity, compileQuery } from '../../pure/messages'; -import { DatabaseItem } from '../../databases'; +import { QueryEvaluationInfo, queriesDir } from '../../run-queries'; +import { Severity, compileQuery } from '../../pure/messages'; +import { Uri } from 'vscode'; chai.use(chaiAsPromised); const expect = chai.expect; @@ -16,29 +16,28 @@ describe('run-queries', () => { it('should create a QueryEvaluationInfo', () => { const info = createMockQueryInfo(); - const queryID = info.queryID; - expect(path.basename(info.compiledQueryPath)).to.eq(`compiledQuery${queryID}.qlo`); - expect(path.basename(info.dilPath)).to.eq(`results${queryID}.dil`); - expect(path.basename(info.resultsPaths.resultsPath)).to.eq(`results${queryID}.bqrs`); - expect(path.basename(info.resultsPaths.interpretedResultsPath)).to.eq(`interpretedResults${queryID}.sarif`); - expect(info.dataset).to.eq('file:///abc'); + const queryId = info.id; + expect(info.compiledQueryPath).to.eq(path.join(queriesDir, queryId, 'compiledQuery.qlo')); + expect(info.dilPath).to.eq(path.join(queriesDir, queryId, 'results.dil')); + expect(info.resultsPaths.resultsPath).to.eq(path.join(queriesDir, queryId, 'results.bqrs')); + expect(info.resultsPaths.interpretedResultsPath).to.eq(path.join(queriesDir, queryId, 'interpretedResults.sarif')); + expect(info.dbItemPath).to.eq(Uri.file('/abc').fsPath); }); it('should check if interpreted results can be created', async () => { - const info = createMockQueryInfo(); - (info.dbItem.hasMetadataFile as sinon.SinonStub).returns(true); + const info = createMockQueryInfo(true); - expect(await info.canHaveInterpretedResults()).to.eq(true); + expect(info.canHaveInterpretedResults()).to.eq(true); - (info.dbItem.hasMetadataFile as sinon.SinonStub).returns(false); - expect(await info.canHaveInterpretedResults()).to.eq(false); + (info as any).databaseHasMetadataFile = false; + expect(info.canHaveInterpretedResults()).to.eq(false); - (info.dbItem.hasMetadataFile as sinon.SinonStub).returns(true); + (info as any).databaseHasMetadataFile = true; info.metadata!.kind = undefined; - expect(await info.canHaveInterpretedResults()).to.eq(false); + expect(info.canHaveInterpretedResults()).to.eq(false); info.metadata!.kind = 'table'; - expect(await info.canHaveInterpretedResults()).to.eq(false); + expect(info.canHaveInterpretedResults()).to.eq(false); }); describe('compile', () => { @@ -47,9 +46,15 @@ describe('run-queries', () => { const qs = createMockQueryServerClient(); const mockProgress = 'progress-monitor'; const mockCancel = 'cancel-token'; + const mockQlProgram = { + dbschemePath: '', + libraryPath: [], + queryPath: '' + }; const results = await info.compile( qs as any, + mockQlProgram, mockProgress as any, mockCancel as any ); @@ -74,7 +79,7 @@ describe('run-queries', () => { extraOptions: { timeoutSecs: 5 }, - queryToCheck: 'my-program', + queryToCheck: mockQlProgram, resultPath: info.compiledQueryPath, target: { query: {} } }, @@ -85,16 +90,11 @@ describe('run-queries', () => { }); let queryNum = 0; - function createMockQueryInfo() { + function createMockQueryInfo(databaseHasMetadataFile = true) { return new QueryEvaluationInfo( - queryNum++, - 'my-program' as unknown as QlProgram, - { - contents: { - datasetUri: 'file:///abc' - }, - hasMetadataFile: sinon.stub() - } as unknown as DatabaseItem, + `save-dir${queryNum++}`, + Uri.parse('file:///abc').fsPath, + databaseHasMetadataFile, 'my-scheme', // queryDbscheme, undefined, {