From 45d805d33bbfc3aa33bdebaf7a5d1ac96d30a2c7 Mon Sep 17 00:00:00 2001 From: shati-patel <42641846+shati-patel@users.noreply.github.com> Date: Fri, 18 Mar 2022 17:21:15 +0000 Subject: [PATCH 1/8] Download dbs from GitHub --- extensions/ql-vscode/package.json | 25 ++++- extensions/ql-vscode/src/databaseFetcher.ts | 97 ++++++++++++++++++- extensions/ql-vscode/src/databases-ui.ts | 34 ++++++- extensions/ql-vscode/src/extension.ts | 15 ++- .../src/remote-queries/run-remote-query.ts | 2 +- .../no-workspace/databases-ui.test.ts | 4 +- 6 files changed, 170 insertions(+), 7 deletions(-) diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 3fe3d95de18..4f109845d30 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -45,6 +45,7 @@ "onCommand:codeQLDatabases.chooseDatabaseFolder", "onCommand:codeQLDatabases.chooseDatabaseArchive", "onCommand:codeQLDatabases.chooseDatabaseInternet", + "onCommand:codeQLDatabases.chooseDatabaseGithub", "onCommand:codeQLDatabases.chooseDatabaseLgtm", "onCommand:codeQL.setCurrentDatabase", "onCommand:codeQL.viewAst", @@ -54,6 +55,7 @@ "onCommand:codeQL.chooseDatabaseFolder", "onCommand:codeQL.chooseDatabaseArchive", "onCommand:codeQL.chooseDatabaseInternet", + "onCommand:codeQL.chooseDatabaseGithub", "onCommand:codeQL.chooseDatabaseLgtm", "onCommand:codeQLDatabases.chooseDatabase", "onCommand:codeQLDatabases.setCurrentDatabase", @@ -356,6 +358,14 @@ "dark": "media/dark/cloud-download.svg" } }, + { + "command": "codeQLDatabases.chooseDatabaseGithub", + "title": "Download Database from GitHub", + "icon": { + "light": "media/light/cloud-download.svg", + "dark": "media/dark/cloud-download.svg" + } + }, { "command": "codeQLDatabases.chooseDatabaseLgtm", "title": "Download from LGTM", @@ -428,6 +438,10 @@ "command": "codeQL.chooseDatabaseInternet", "title": "CodeQL: Download Database" }, + { + "command": "codeQL.chooseDatabaseGithub", + "title": "CodeQL: Download Database from GitHub" + }, { "command": "codeQL.chooseDatabaseLgtm", "title": "CodeQL: Download Database from LGTM" @@ -604,6 +618,11 @@ "when": "view == codeQLDatabases", "group": "navigation" }, + { + "command": "codeQLDatabases.chooseDatabaseGithub", + "when": "view == codeQLDatabases", + "group": "navigation" + }, { "command": "codeQLDatabases.chooseDatabaseLgtm", "when": "view == codeQLDatabases", @@ -873,6 +892,10 @@ "command": "codeQLDatabases.chooseDatabaseInternet", "when": "false" }, + { + "command": "codeQLDatabases.chooseDatabaseGithub", + "when": "false" + }, { "command": "codeQLDatabases.chooseDatabaseLgtm", "when": "false" @@ -1037,7 +1060,7 @@ }, { "view": "codeQLDatabases", - "contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)" + "contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From GitHub](command:codeQLDatabases.chooseDatabaseGithub)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)" } ] }, diff --git a/extensions/ql-vscode/src/databaseFetcher.ts b/extensions/ql-vscode/src/databaseFetcher.ts index 6f64b0aa7ce..e74b2024ab2 100644 --- a/extensions/ql-vscode/src/databaseFetcher.ts +++ b/extensions/ql-vscode/src/databaseFetcher.ts @@ -21,6 +21,8 @@ import { } from './commandRunner'; import { logger } from './logging'; import { tmpDir } from './helpers'; +import { REPO_REGEX } from './remote-queries/run-remote-query'; +import { Credentials } from './authentication'; /** * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file. @@ -46,6 +48,7 @@ export async function promptImportInternetDatabase( const item = await databaseArchiveFetcher( databaseUrl, + {}, databaseManager, storagePath, progress, @@ -61,6 +64,60 @@ export async function promptImportInternetDatabase( } +/** + * Prompts a user to fetch a database from GitHub. + * User enters a GitHub repository and then the user is asked which language + * to download (if there is more than one) + * + * @param databaseManager the DatabaseManager + * @param storagePath where to store the unzipped database. + */ +export async function promptImportGithubDatabase( + databaseManager: DatabaseManager, + storagePath: string, + credentials: Credentials, + progress: ProgressCallback, + token: CancellationToken, + cli?: CodeQLCliServer +): Promise { + progress({ + message: 'Choose repository', + step: 1, + maxStep: 2 + }); + const githubRepo = await window.showInputBox({ + title: 'Enter a GitHub repository in the format / (e.g. github/codeql)', + placeHolder: '/', + ignoreFocusOut: true, + }); + if (!githubRepo) { + return; + } + + if (REPO_REGEX.test(githubRepo)) { + const databaseUrl = await convertGithubNwoToDatabaseUrl(githubRepo, credentials, progress); + if (databaseUrl) { + const item = await databaseArchiveFetcher( + databaseUrl, + { 'Accept': 'application/zip' }, + databaseManager, + storagePath, + progress, + token, + cli + ); + if (item) { + await commands.executeCommand('codeQLDatabases.focus'); + void showAndLogInformationMessage('Database downloaded and imported successfully.'); + } + return item; + } + } else { + throw new Error(`Invalid GitHub repository: ${githubRepo}`); + } + return; +} + /** * Prompts a user to fetch a database from lgtm. * User enters a project url and then the user is asked which language @@ -94,6 +151,7 @@ export async function promptImportLgtmDatabase( if (databaseUrl) { const item = await databaseArchiveFetcher( databaseUrl, + {}, databaseManager, storagePath, progress, @@ -140,6 +198,7 @@ export async function importArchiveDatabase( try { const item = await databaseArchiveFetcher( databaseUrl, + {}, databaseManager, storagePath, progress, @@ -166,6 +225,7 @@ export async function importArchiveDatabase( * or in the local filesystem. * * @param databaseUrl URL from which to grab the database + * @param requestHeaders Headers to send with the request * @param databaseManager the DatabaseManager * @param storagePath where to store the unzipped database. * @param progress callback to send progress messages to @@ -173,6 +233,7 @@ export async function importArchiveDatabase( */ async function databaseArchiveFetcher( databaseUrl: string, + requestHeaders: { [key: string]: string }, databaseManager: DatabaseManager, storagePath: string, progress: ProgressCallback, @@ -193,7 +254,7 @@ async function databaseArchiveFetcher( if (isFile(databaseUrl)) { await readAndUnzip(databaseUrl, unzipPath, cli, progress); } else { - await fetchAndUnzip(databaseUrl, unzipPath, cli, progress); + await fetchAndUnzip(databaseUrl, requestHeaders, unzipPath, cli, progress); } progress({ @@ -292,6 +353,7 @@ async function readAndUnzip( async function fetchAndUnzip( databaseUrl: string, + requestHeaders: { [key: string]: string }, unzipPath: string, cli?: CodeQLCliServer, progress?: ProgressCallback @@ -310,7 +372,7 @@ async function fetchAndUnzip( step: 1, }); - const response = await checkForFailingResponse(await fetch(databaseUrl), 'Error downloading database'); + const response = await checkForFailingResponse(await fetch(databaseUrl, requestHeaders), 'Error downloading database'); const archiveFileStream = fs.createWriteStream(archivePath); const contentLength = response.headers.get('content-length'); @@ -381,6 +443,37 @@ export async function findDirWithFile( return; } +export async function convertGithubNwoToDatabaseUrl( + githubRepo: string, + credentials: Credentials, + progress: ProgressCallback): Promise { + try { + // TODO: In future, we could accept GitHub URLs in addition to NWOs. + // Similar to "looksLikeLgtmUrl". + if (!REPO_REGEX.test(githubRepo)) { + throw new Error('Invalid repository format. Must be in the format / (e.g. github/codeql)'); + } + + const [owner, repo] = githubRepo.split('/'); + + const octokit = await credentials.getOctokit(); + const response = await octokit.request('GET /repos/:owner/:repo/code-scanning/codeql/databases', { owner, repo }); + + const languages = response.data.map((db: any) => db.language); + + const language = await promptForLanguage(languages, progress); + if (!language) { + return; + } + + return `https://api.github.com/repos/${owner}/${repo}/code-scanning/codeql/databases/${language}`; + + } catch (e) { + void logger.log(`Error: ${e.message}`); + throw new Error(`Invalid GitHub repository: ${githubRepo}`); + } +} + /** * The URL pattern is https://lgtm.com/projects/{provider}/{org}/{name}/{irrelevant-subpages}. * There are several possibilities for the provider: in addition to GitHub.com (g), diff --git a/extensions/ql-vscode/src/databases-ui.ts b/extensions/ql-vscode/src/databases-ui.ts index 7c4aeecd020..ee81b568f8c 100644 --- a/extensions/ql-vscode/src/databases-ui.ts +++ b/extensions/ql-vscode/src/databases-ui.ts @@ -33,11 +33,13 @@ import * as qsClient from './queryserver-client'; import { upgradeDatabaseExplicit } from './upgrades'; import { importArchiveDatabase, + promptImportGithubDatabase, promptImportInternetDatabase, promptImportLgtmDatabase, } from './databaseFetcher'; import { CancellationToken } from 'vscode'; import { asyncFilter } from './pure/helpers-pure'; +import { Credentials } from './authentication'; type ThemableIconPath = { light: string; dark: string } | string; @@ -219,7 +221,8 @@ export class DatabaseUI extends DisposableObject { private databaseManager: DatabaseManager, private readonly queryServer: qsClient.QueryServerClient | undefined, private readonly storagePath: string, - readonly extensionPath: string + readonly extensionPath: string, + private readonly getCredentials: () => Promise ) { super(); @@ -291,6 +294,20 @@ export class DatabaseUI extends DisposableObject { } ) ); + this.push( + commandRunnerWithProgress( + 'codeQLDatabases.chooseDatabaseGithub', + async ( + progress: ProgressCallback, + token: CancellationToken + ) => { + const credentials = await this.getCredentials(); + await this.handleChooseDatabaseGithub(credentials, progress, token); + }, + { + title: 'Adding database from GitHub', + }) + ); this.push( commandRunnerWithProgress( 'codeQLDatabases.chooseDatabaseLgtm', @@ -462,6 +479,21 @@ export class DatabaseUI extends DisposableObject { ); }; + handleChooseDatabaseGithub = async ( + credentials: Credentials, + progress: ProgressCallback, + token: CancellationToken + ): Promise => { + return await promptImportGithubDatabase( + this.databaseManager, + this.storagePath, + credentials, + progress, + token, + this.queryServer?.cliServer + ); + }; + handleChooseDatabaseLgtm = async ( progress: ProgressCallback, token: CancellationToken diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts index 6626b3f7618..39cca56ca9e 100644 --- a/extensions/ql-vscode/src/extension.ts +++ b/extensions/ql-vscode/src/extension.ts @@ -433,7 +433,8 @@ async function activateWithInstalledDistribution( dbm, qs, getContextStoragePath(ctx), - ctx.extensionPath + ctx.extensionPath, + () => Credentials.initialize(ctx), ); databaseUI.init(); ctx.subscriptions.push(databaseUI); @@ -931,6 +932,18 @@ async function activateWithInstalledDistribution( title: 'Choose a Database from an Archive' }) ); + ctx.subscriptions.push( + commandRunnerWithProgress('codeQL.chooseDatabaseGithub', async ( + progress: ProgressCallback, + token: CancellationToken + ) => { + const credentials = await Credentials.initialize(ctx); + await databaseUI.handleChooseDatabaseGithub(credentials, progress, token); + }, + { + title: 'Adding database from GitHub', + }) + ); ctx.subscriptions.push( commandRunnerWithProgress('codeQL.chooseDatabaseLgtm', ( progress: ProgressCallback, diff --git a/extensions/ql-vscode/src/remote-queries/run-remote-query.ts b/extensions/ql-vscode/src/remote-queries/run-remote-query.ts index d59c0fdfa21..eb84170fc91 100644 --- a/extensions/ql-vscode/src/remote-queries/run-remote-query.ts +++ b/extensions/ql-vscode/src/remote-queries/run-remote-query.ts @@ -43,7 +43,7 @@ interface QueriesResponse { * - `owner` is made up of alphanumeric characters or single hyphens, starting and ending in an alphanumeric character * - `repo` is made up of alphanumeric characters, hyphens, or underscores */ -const REPO_REGEX = /^(?:[a-zA-Z0-9]+-)*[a-zA-Z0-9]+\/[a-zA-Z0-9-_]+$/; +export const REPO_REGEX = /^(?:[a-zA-Z0-9]+-)*[a-zA-Z0-9]+\/[a-zA-Z0-9-_]+$/; /** * Well-known names for the query pack used by the server. diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/databases-ui.test.ts b/extensions/ql-vscode/src/vscode-tests/no-workspace/databases-ui.test.ts index 685c291860f..656a57f9572 100644 --- a/extensions/ql-vscode/src/vscode-tests/no-workspace/databases-ui.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/databases-ui.test.ts @@ -8,6 +8,7 @@ import { Uri } from 'vscode'; import { DatabaseUI } from '../../databases-ui'; import { testDisposeHandler } from '../test-dispose-handler'; +import { Credentials } from '../../authentication'; describe('databases-ui', () => { describe('fixDbUri', () => { @@ -78,7 +79,8 @@ describe('databases-ui', () => { } as any, {} as any, storageDir, - storageDir + storageDir, + () => Promise.resolve({} as Credentials), ); await databaseUI.handleRemoveOrphanedDatabases(); From bc727d5bad37affee9efe8f5d3ed454f22707f69 Mon Sep 17 00:00:00 2001 From: shati-patel <42641846+shati-patel@users.noreply.github.com> Date: Tue, 22 Mar 2022 14:09:51 +0000 Subject: [PATCH 2/8] Pass in credentials for authenticating to GitHub API --- extensions/ql-vscode/src/authentication.ts | 43 +++++++++++++++------ extensions/ql-vscode/src/databaseFetcher.ts | 8 +++- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/extensions/ql-vscode/src/authentication.ts b/extensions/ql-vscode/src/authentication.ts index 7c226f526ec..ac21b7f9e02 100644 --- a/extensions/ql-vscode/src/authentication.ts +++ b/extensions/ql-vscode/src/authentication.ts @@ -7,11 +7,16 @@ const GITHUB_AUTH_PROVIDER_ID = 'github'; // https://docs.github.com/apps/building-oauth-apps/understanding-scopes-for-oauth-apps const SCOPES = ['repo']; +interface OctokitAndToken { + octokit: Octokit.Octokit; + token: string; +} + /** * Handles authentication to GitHub, using the VS Code [authentication API](https://code.visualstudio.com/api/references/vscode-api#authentication). */ export class Credentials { - private octokit: Octokit.Octokit | undefined; + private octokitAndToken: OctokitAndToken | undefined; // Explicitly make the constructor private, so that we can't accidentally call the constructor from outside the class // without also initializing the class. @@ -21,17 +26,20 @@ export class Credentials { static async initialize(context: vscode.ExtensionContext): Promise { const c = new Credentials(); c.registerListeners(context); - c.octokit = await c.createOctokit(false); + c.octokitAndToken = await c.createOctokit(false); return c; } - private async createOctokit(createIfNone: boolean): Promise { + private async createOctokit(createIfNone: boolean): Promise { const session = await vscode.authentication.getSession(GITHUB_AUTH_PROVIDER_ID, SCOPES, { createIfNone }); if (session) { - return new Octokit.Octokit({ - auth: session.accessToken - }); + return { + octokit: new Octokit.Octokit({ + auth: session.accessToken + }), + token: session.accessToken + }; } else { return undefined; } @@ -41,22 +49,33 @@ export class Credentials { // Sessions are changed when a user logs in or logs out. context.subscriptions.push(vscode.authentication.onDidChangeSessions(async e => { if (e.provider.id === GITHUB_AUTH_PROVIDER_ID) { - this.octokit = await this.createOctokit(false); + this.octokitAndToken = await this.createOctokit(false); } })); } async getOctokit(): Promise { - if (this.octokit) { - return this.octokit; + if (this.octokitAndToken) { + return this.octokitAndToken.octokit; } - this.octokit = await this.createOctokit(true); + this.octokitAndToken = await this.createOctokit(true); // octokit shouldn't be undefined, since we've set "createIfNone: true". // The following block is mainly here to prevent a compiler error. - if (!this.octokit) { + if (!this.octokitAndToken) { + throw new Error('Did not initialize Octokit.'); + } + return this.octokitAndToken.octokit; + } + + async getToken(): Promise { + if (this.octokitAndToken) { + return this.octokitAndToken.token; + } + this.octokitAndToken = await this.createOctokit(true); + if (!this.octokitAndToken) { throw new Error('Did not initialize Octokit.'); } - return this.octokit; + return this.octokitAndToken.token; } } diff --git a/extensions/ql-vscode/src/databaseFetcher.ts b/extensions/ql-vscode/src/databaseFetcher.ts index e74b2024ab2..9f2013d9d49 100644 --- a/extensions/ql-vscode/src/databaseFetcher.ts +++ b/extensions/ql-vscode/src/databaseFetcher.ts @@ -97,9 +97,10 @@ export async function promptImportGithubDatabase( if (REPO_REGEX.test(githubRepo)) { const databaseUrl = await convertGithubNwoToDatabaseUrl(githubRepo, credentials, progress); if (databaseUrl) { + const octokitToken = await credentials.getToken(); const item = await databaseArchiveFetcher( databaseUrl, - { 'Accept': 'application/zip' }, + { 'Accept': 'application/zip', 'Authorization': `Bearer ${octokitToken}` }, databaseManager, storagePath, progress, @@ -372,7 +373,10 @@ async function fetchAndUnzip( step: 1, }); - const response = await checkForFailingResponse(await fetch(databaseUrl, requestHeaders), 'Error downloading database'); + const response = await checkForFailingResponse( + await fetch(databaseUrl, { headers: requestHeaders }), + 'Error downloading database' + ); const archiveFileStream = fs.createWriteStream(archivePath); const contentLength = response.headers.get('content-length'); From 18bfd67ceb8a16ec11761bcfa1168aba3e77e312 Mon Sep 17 00:00:00 2001 From: shati-patel <42641846+shati-patel@users.noreply.github.com> Date: Wed, 23 Mar 2022 12:51:15 +0000 Subject: [PATCH 3/8] Use GitHub icon in Databases UI From https://github.com/microsoft/vscode-icons --- extensions/ql-vscode/media/dark/github.svg | 3 +++ extensions/ql-vscode/media/light/github.svg | 10 ++++++++++ extensions/ql-vscode/package.json | 4 ++-- 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 extensions/ql-vscode/media/dark/github.svg create mode 100644 extensions/ql-vscode/media/light/github.svg diff --git a/extensions/ql-vscode/media/dark/github.svg b/extensions/ql-vscode/media/dark/github.svg new file mode 100644 index 00000000000..51c34396a94 --- /dev/null +++ b/extensions/ql-vscode/media/dark/github.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/extensions/ql-vscode/media/light/github.svg b/extensions/ql-vscode/media/light/github.svg new file mode 100644 index 00000000000..5439738300f --- /dev/null +++ b/extensions/ql-vscode/media/light/github.svg @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 4f109845d30..54567726b11 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -362,8 +362,8 @@ "command": "codeQLDatabases.chooseDatabaseGithub", "title": "Download Database from GitHub", "icon": { - "light": "media/light/cloud-download.svg", - "dark": "media/dark/cloud-download.svg" + "light": "media/light/github.svg", + "dark": "media/dark/github.svg" } }, { From 9c21b6622761b9ff3c248b3c06c36172a4e171ba Mon Sep 17 00:00:00 2001 From: shati-patel <42641846+shati-patel@users.noreply.github.com> Date: Wed, 23 Mar 2022 14:31:09 +0000 Subject: [PATCH 4/8] Get octokit token using `.auth()` --- extensions/ql-vscode/src/authentication.ts | 43 +++++----------- extensions/ql-vscode/src/databaseFetcher.ts | 55 +++++++++++++-------- 2 files changed, 47 insertions(+), 51 deletions(-) diff --git a/extensions/ql-vscode/src/authentication.ts b/extensions/ql-vscode/src/authentication.ts index ac21b7f9e02..7c226f526ec 100644 --- a/extensions/ql-vscode/src/authentication.ts +++ b/extensions/ql-vscode/src/authentication.ts @@ -7,16 +7,11 @@ const GITHUB_AUTH_PROVIDER_ID = 'github'; // https://docs.github.com/apps/building-oauth-apps/understanding-scopes-for-oauth-apps const SCOPES = ['repo']; -interface OctokitAndToken { - octokit: Octokit.Octokit; - token: string; -} - /** * Handles authentication to GitHub, using the VS Code [authentication API](https://code.visualstudio.com/api/references/vscode-api#authentication). */ export class Credentials { - private octokitAndToken: OctokitAndToken | undefined; + private octokit: Octokit.Octokit | undefined; // Explicitly make the constructor private, so that we can't accidentally call the constructor from outside the class // without also initializing the class. @@ -26,20 +21,17 @@ export class Credentials { static async initialize(context: vscode.ExtensionContext): Promise { const c = new Credentials(); c.registerListeners(context); - c.octokitAndToken = await c.createOctokit(false); + c.octokit = await c.createOctokit(false); return c; } - private async createOctokit(createIfNone: boolean): Promise { + private async createOctokit(createIfNone: boolean): Promise { const session = await vscode.authentication.getSession(GITHUB_AUTH_PROVIDER_ID, SCOPES, { createIfNone }); if (session) { - return { - octokit: new Octokit.Octokit({ - auth: session.accessToken - }), - token: session.accessToken - }; + return new Octokit.Octokit({ + auth: session.accessToken + }); } else { return undefined; } @@ -49,33 +41,22 @@ export class Credentials { // Sessions are changed when a user logs in or logs out. context.subscriptions.push(vscode.authentication.onDidChangeSessions(async e => { if (e.provider.id === GITHUB_AUTH_PROVIDER_ID) { - this.octokitAndToken = await this.createOctokit(false); + this.octokit = await this.createOctokit(false); } })); } async getOctokit(): Promise { - if (this.octokitAndToken) { - return this.octokitAndToken.octokit; + if (this.octokit) { + return this.octokit; } - this.octokitAndToken = await this.createOctokit(true); + this.octokit = await this.createOctokit(true); // octokit shouldn't be undefined, since we've set "createIfNone: true". // The following block is mainly here to prevent a compiler error. - if (!this.octokitAndToken) { - throw new Error('Did not initialize Octokit.'); - } - return this.octokitAndToken.octokit; - } - - async getToken(): Promise { - if (this.octokitAndToken) { - return this.octokitAndToken.token; - } - this.octokitAndToken = await this.createOctokit(true); - if (!this.octokitAndToken) { + if (!this.octokit) { throw new Error('Did not initialize Octokit.'); } - return this.octokitAndToken.token; + return this.octokit; } } diff --git a/extensions/ql-vscode/src/databaseFetcher.ts b/extensions/ql-vscode/src/databaseFetcher.ts index 9f2013d9d49..b2ff8ac0d68 100644 --- a/extensions/ql-vscode/src/databaseFetcher.ts +++ b/extensions/ql-vscode/src/databaseFetcher.ts @@ -94,28 +94,43 @@ export async function promptImportGithubDatabase( return; } - if (REPO_REGEX.test(githubRepo)) { - const databaseUrl = await convertGithubNwoToDatabaseUrl(githubRepo, credentials, progress); - if (databaseUrl) { - const octokitToken = await credentials.getToken(); - const item = await databaseArchiveFetcher( - databaseUrl, - { 'Accept': 'application/zip', 'Authorization': `Bearer ${octokitToken}` }, - databaseManager, - storagePath, - progress, - token, - cli - ); - if (item) { - await commands.executeCommand('codeQLDatabases.focus'); - void showAndLogInformationMessage('Database downloaded and imported successfully.'); - } - return item; - } - } else { + if (!REPO_REGEX.test(githubRepo)) { throw new Error(`Invalid GitHub repository: ${githubRepo}`); } + + const databaseUrl = await convertGithubNwoToDatabaseUrl(githubRepo, credentials, progress); + if (databaseUrl) { + const octokit = await credentials.getOctokit(); + /** + * A token object returned by `octokit.auth()`. It's undocumented, but looks something like this: + * { + * token: 'xxxx', + * tokenType: 'oauth', + * type: 'token', + * } + * We only need the actual token string. + */ + const octokitToken = await octokit.auth() as { token: string }; + if (!octokitToken.token) { + // Just print a generic error message for now. Ideally we could show more debugging info, like the + // octokit object, but that would expose a user token. + throw new Error('Unable to get GitHub token.'); + } + const item = await databaseArchiveFetcher( + databaseUrl, + { 'Accept': 'application/zip', 'Authorization': `Bearer ${octokitToken.token}` }, + databaseManager, + storagePath, + progress, + token, + cli + ); + if (item) { + await commands.executeCommand('codeQLDatabases.focus'); + void showAndLogInformationMessage('Database downloaded and imported successfully.'); + } + return item; + } return; } From b5a0ee6c1754eaf35d071b4c11c9b550964dc601 Mon Sep 17 00:00:00 2001 From: shati-patel <42641846+shati-patel@users.noreply.github.com> Date: Wed, 23 Mar 2022 15:11:42 +0000 Subject: [PATCH 5/8] =?UTF-8?q?Hide=20"Download=20db=20from=20GH"=20behind?= =?UTF-8?q?=20canary=20flag=20=F0=9F=A4=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extensions/ql-vscode/package.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 54567726b11..22cb67af97c 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -620,7 +620,7 @@ }, { "command": "codeQLDatabases.chooseDatabaseGithub", - "when": "view == codeQLDatabases", + "when": "config.codeQL.canary && view == codeQLDatabases", "group": "navigation" }, { @@ -848,6 +848,10 @@ "command": "codeQL.viewCfg", "when": "resourceScheme == codeql-zip-archive && config.codeQL.canary" }, + { + "command": "codeQL.chooseDatabaseGithub", + "when": "config.codeQL.canary" + }, { "command": "codeQLDatabases.setCurrentDatabase", "when": "false" @@ -1060,7 +1064,7 @@ }, { "view": "codeQLDatabases", - "contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From GitHub](command:codeQLDatabases.chooseDatabaseGithub)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)" + "contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)" } ] }, From d7a1e426834a9004cb84570a11a1f85aa50227e0 Mon Sep 17 00:00:00 2001 From: shati-patel <42641846+shati-patel@users.noreply.github.com> Date: Thu, 24 Mar 2022 12:25:57 +0000 Subject: [PATCH 6/8] Add tests --- extensions/ql-vscode/src/databaseFetcher.ts | 4 +- .../no-workspace/databaseFetcher.test.ts | 129 ++++++++++++++++++ 2 files changed, 131 insertions(+), 2 deletions(-) diff --git a/extensions/ql-vscode/src/databaseFetcher.ts b/extensions/ql-vscode/src/databaseFetcher.ts index b2ff8ac0d68..4d6303ba012 100644 --- a/extensions/ql-vscode/src/databaseFetcher.ts +++ b/extensions/ql-vscode/src/databaseFetcher.ts @@ -489,7 +489,7 @@ export async function convertGithubNwoToDatabaseUrl( } catch (e) { void logger.log(`Error: ${e.message}`); - throw new Error(`Invalid GitHub repository: ${githubRepo}`); + throw new Error(`Unable to get database for '${githubRepo}'`); } } @@ -618,7 +618,7 @@ async function promptForLanguage( maxStep: 2 }); if (!languages.length) { - return; + throw new Error('No databases found'); } if (languages.length === 1) { return languages[0]; diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts b/extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts index 374bcb2f61f..1da22b6cdef 100644 --- a/extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts @@ -14,6 +14,9 @@ import { findDirWithFile, } from '../../databaseFetcher'; import { ProgressCallback } from '../../commandRunner'; +import * as pq from 'proxyquire'; + +const proxyquire = pq.noPreserveCache(); chai.use(chaiAsPromised); const expect = chai.expect; @@ -21,6 +24,132 @@ describe('databaseFetcher', function() { // These tests make API calls and may need extra time to complete. this.timeout(10000); + describe('convertGithubNwoToDatabaseUrl', () => { + let sandbox: sinon.SinonSandbox; + let quickPickSpy: sinon.SinonStub; + let progressSpy: ProgressCallback; + let mockRequest: sinon.SinonStub; + let mod: any; + + const credentials = getMockCredentials(0); + + beforeEach(() => { + sandbox = sinon.createSandbox(); + quickPickSpy = sandbox.stub(window, 'showQuickPick'); + progressSpy = sandbox.spy(); + mockRequest = sandbox.stub(); + mod = proxyquire('../../databaseFetcher', { + './authentication': { + Credentials: credentials, + }, + }); + }); + + afterEach(() => { + sandbox.restore(); + }); + + it('should convert a GitHub nwo to a database url', async () => { + // We can't make the real octokit request (since we need credentials), so we mock the response. + const mockApiResponse = { + data: [ + { + id: 1495869, + name: 'csharp-database', + language: 'csharp', + uploader: {}, + content_type: 'application/zip', + state: 'uploaded', + size: 55599715, + created_at: '2022-03-24T10:46:24Z', + updated_at: '2022-03-24T10:46:27Z', + url: 'https://api.github.com/repositories/143040428/code-scanning/codeql/databases/csharp', + }, + { + id: 1100671, + name: 'database.zip', + language: 'javascript', + uploader: {}, + content_type: 'application/zip', + state: 'uploaded', + size: 29294434, + created_at: '2022-03-01T16:00:04Z', + updated_at: '2022-03-01T16:00:06Z', + url: 'https://api.github.com/repositories/143040428/code-scanning/codeql/databases/javascript', + }, + { + id: 648738, + name: 'ql-database', + language: 'ql', + uploader: {}, + content_type: 'application/json; charset=utf-8', + state: 'uploaded', + size: 39735500, + created_at: '2022-02-02T09:38:50Z', + updated_at: '2022-02-02T09:38:51Z', + url: 'https://api.github.com/repositories/143040428/code-scanning/codeql/databases/ql', + }, + ], + }; + mockRequest.resolves(mockApiResponse); + quickPickSpy.resolves('javascript'); + const githubRepo = 'github/codeql'; + const dbUrl = await mod.convertGithubNwoToDatabaseUrl( + githubRepo, + credentials, + progressSpy + ); + + expect(dbUrl).to.equal( + 'https://api.github.com/repos/github/codeql/code-scanning/codeql/databases/javascript' + ); + expect(quickPickSpy.firstCall.args[0]).to.deep.equal([ + 'csharp', + 'javascript', + 'ql', + ]); + }); + + // Repository doesn't exist, or the user has no access to the repository. + it('should fail on an invalid/inaccessible repository', async () => { + const mockApiResponse = { + data: { + message: 'Not Found', + }, + status: 404, + }; + mockRequest.resolves(mockApiResponse); + const githubRepo = 'foo/bar-not-real'; + await expect( + mod.convertGithubNwoToDatabaseUrl(githubRepo, credentials, progressSpy) + ).to.be.rejectedWith(/Unable to get database/); + expect(progressSpy).to.have.callCount(0); + }); + + // User has access to the repository, but there are no databases for any language. + it('should fail on a repository with no databases', async () => { + const mockApiResponse = { + data: [], + }; + + mockRequest.resolves(mockApiResponse); + const githubRepo = 'foo/bar-with-no-dbs'; + await expect( + mod.convertGithubNwoToDatabaseUrl(githubRepo, credentials, progressSpy) + ).to.be.rejectedWith(/Unable to get database/); + expect(progressSpy).to.have.been.calledOnce; + }); + + function getMockCredentials(response: any) { + mockRequest = sinon.stub().resolves(response); + return { + getOctokit: () => ({ + request: mockRequest, + }), + }; + } + }); + describe('convertLgtmUrlToDatabaseUrl', () => { let sandbox: sinon.SinonSandbox; let quickPickSpy: sinon.SinonStub; From ed7b285f487ba2a7c09017e54f989227ec4b271f Mon Sep 17 00:00:00 2001 From: shati-patel <42641846+shati-patel@users.noreply.github.com> Date: Thu, 24 Mar 2022 18:07:24 +0000 Subject: [PATCH 7/8] Move REPO_REGEX into helper file --- extensions/ql-vscode/src/databaseFetcher.ts | 2 +- extensions/ql-vscode/src/pure/helpers-pure.ts | 7 +++++++ .../ql-vscode/src/remote-queries/run-remote-query.ts | 8 +------- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/extensions/ql-vscode/src/databaseFetcher.ts b/extensions/ql-vscode/src/databaseFetcher.ts index 4d6303ba012..6d2ac54e453 100644 --- a/extensions/ql-vscode/src/databaseFetcher.ts +++ b/extensions/ql-vscode/src/databaseFetcher.ts @@ -21,8 +21,8 @@ import { } from './commandRunner'; import { logger } from './logging'; import { tmpDir } from './helpers'; -import { REPO_REGEX } from './remote-queries/run-remote-query'; import { Credentials } from './authentication'; +import { REPO_REGEX } from './pure/helpers-pure'; /** * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file. diff --git a/extensions/ql-vscode/src/pure/helpers-pure.ts b/extensions/ql-vscode/src/pure/helpers-pure.ts index 3940344b658..1c98b59e49f 100644 --- a/extensions/ql-vscode/src/pure/helpers-pure.ts +++ b/extensions/ql-vscode/src/pure/helpers-pure.ts @@ -35,3 +35,10 @@ export const ONE_DAY_IN_MS = 24 * 60 * 60 * 1000; export const ONE_HOUR_IN_MS = 1000 * 60 * 60; export const TWO_HOURS_IN_MS = 1000 * 60 * 60 * 2; export const THREE_HOURS_IN_MS = 1000 * 60 * 60 * 3; + +/** + * This regex matches strings of the form `owner/repo` where: + * - `owner` is made up of alphanumeric characters or single hyphens, starting and ending in an alphanumeric character + * - `repo` is made up of alphanumeric characters, hyphens, or underscores + */ +export const REPO_REGEX = /^(?:[a-zA-Z0-9]+-)*[a-zA-Z0-9]+\/[a-zA-Z0-9-_]+$/; diff --git a/extensions/ql-vscode/src/remote-queries/run-remote-query.ts b/extensions/ql-vscode/src/remote-queries/run-remote-query.ts index eb84170fc91..b509f68ce0d 100644 --- a/extensions/ql-vscode/src/remote-queries/run-remote-query.ts +++ b/extensions/ql-vscode/src/remote-queries/run-remote-query.ts @@ -22,6 +22,7 @@ import { OctokitResponse } from '@octokit/types/dist-types'; import { RemoteQuery } from './remote-query'; import { RemoteQuerySubmissionResult } from './remote-query-submission-result'; import { QueryMetadata } from '../pure/interface-types'; +import { REPO_REGEX } from '../pure/helpers-pure'; export interface QlPack { name: string; @@ -38,13 +39,6 @@ interface QueriesResponse { workflow_run_id: number } -/** - * This regex matches strings of the form `owner/repo` where: - * - `owner` is made up of alphanumeric characters or single hyphens, starting and ending in an alphanumeric character - * - `repo` is made up of alphanumeric characters, hyphens, or underscores - */ -export const REPO_REGEX = /^(?:[a-zA-Z0-9]+-)*[a-zA-Z0-9]+\/[a-zA-Z0-9-_]+$/; - /** * Well-known names for the query pack used by the server. */ From 1bf8cc6549f7142a53efc81c6f4bc25fc4655c30 Mon Sep 17 00:00:00 2001 From: shati-patel <42641846+shati-patel@users.noreply.github.com> Date: Fri, 25 Mar 2022 11:00:22 +0000 Subject: [PATCH 8/8] Add source to SVG files + address code comments --- extensions/ql-vscode/media/dark/github.svg | 1 + extensions/ql-vscode/media/light/github.svg | 1 + extensions/ql-vscode/src/databaseFetcher.ts | 63 +++++++++++---------- 3 files changed, 35 insertions(+), 30 deletions(-) diff --git a/extensions/ql-vscode/media/dark/github.svg b/extensions/ql-vscode/media/dark/github.svg index 51c34396a94..f3e8e4ad185 100644 --- a/extensions/ql-vscode/media/dark/github.svg +++ b/extensions/ql-vscode/media/dark/github.svg @@ -1,3 +1,4 @@ + \ No newline at end of file diff --git a/extensions/ql-vscode/media/light/github.svg b/extensions/ql-vscode/media/light/github.svg index 5439738300f..6c5c315b7fb 100644 --- a/extensions/ql-vscode/media/light/github.svg +++ b/extensions/ql-vscode/media/light/github.svg @@ -1,3 +1,4 @@ + diff --git a/extensions/ql-vscode/src/databaseFetcher.ts b/extensions/ql-vscode/src/databaseFetcher.ts index 6d2ac54e453..a3c9aa04c44 100644 --- a/extensions/ql-vscode/src/databaseFetcher.ts +++ b/extensions/ql-vscode/src/databaseFetcher.ts @@ -99,36 +99,39 @@ export async function promptImportGithubDatabase( } const databaseUrl = await convertGithubNwoToDatabaseUrl(githubRepo, credentials, progress); - if (databaseUrl) { - const octokit = await credentials.getOctokit(); - /** - * A token object returned by `octokit.auth()`. It's undocumented, but looks something like this: - * { - * token: 'xxxx', - * tokenType: 'oauth', - * type: 'token', - * } - * We only need the actual token string. - */ - const octokitToken = await octokit.auth() as { token: string }; - if (!octokitToken.token) { - // Just print a generic error message for now. Ideally we could show more debugging info, like the - // octokit object, but that would expose a user token. - throw new Error('Unable to get GitHub token.'); - } - const item = await databaseArchiveFetcher( - databaseUrl, - { 'Accept': 'application/zip', 'Authorization': `Bearer ${octokitToken.token}` }, - databaseManager, - storagePath, - progress, - token, - cli - ); - if (item) { - await commands.executeCommand('codeQLDatabases.focus'); - void showAndLogInformationMessage('Database downloaded and imported successfully.'); - } + if (!databaseUrl) { + return; + } + + const octokit = await credentials.getOctokit(); + /** + * The 'token' property of the token object returned by `octokit.auth()`. + * The object is undocumented, but looks something like this: + * { + * token: 'xxxx', + * tokenType: 'oauth', + * type: 'token', + * } + * We only need the actual token string. + */ + const octokitToken = (await octokit.auth() as { token: string })?.token; + if (!octokitToken) { + // Just print a generic error message for now. Ideally we could show more debugging info, like the + // octokit object, but that would expose a user token. + throw new Error('Unable to get GitHub token.'); + } + const item = await databaseArchiveFetcher( + databaseUrl, + { 'Accept': 'application/zip', 'Authorization': `Bearer ${octokitToken}` }, + databaseManager, + storagePath, + progress, + token, + cli + ); + if (item) { + await commands.executeCommand('codeQLDatabases.focus'); + void showAndLogInformationMessage('Database downloaded and imported successfully.'); return item; } return;