diff --git a/extensions/ql-vscode/media/dark/github.svg b/extensions/ql-vscode/media/dark/github.svg new file mode 100644 index 00000000000..f3e8e4ad185 --- /dev/null +++ b/extensions/ql-vscode/media/dark/github.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/extensions/ql-vscode/media/light/github.svg b/extensions/ql-vscode/media/light/github.svg new file mode 100644 index 00000000000..6c5c315b7fb --- /dev/null +++ b/extensions/ql-vscode/media/light/github.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 3fe3d95de18..22cb67af97c 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -45,6 +45,7 @@ "onCommand:codeQLDatabases.chooseDatabaseFolder", "onCommand:codeQLDatabases.chooseDatabaseArchive", "onCommand:codeQLDatabases.chooseDatabaseInternet", + "onCommand:codeQLDatabases.chooseDatabaseGithub", "onCommand:codeQLDatabases.chooseDatabaseLgtm", "onCommand:codeQL.setCurrentDatabase", "onCommand:codeQL.viewAst", @@ -54,6 +55,7 @@ "onCommand:codeQL.chooseDatabaseFolder", "onCommand:codeQL.chooseDatabaseArchive", "onCommand:codeQL.chooseDatabaseInternet", + "onCommand:codeQL.chooseDatabaseGithub", "onCommand:codeQL.chooseDatabaseLgtm", "onCommand:codeQLDatabases.chooseDatabase", "onCommand:codeQLDatabases.setCurrentDatabase", @@ -356,6 +358,14 @@ "dark": "media/dark/cloud-download.svg" } }, + { + "command": "codeQLDatabases.chooseDatabaseGithub", + "title": "Download Database from GitHub", + "icon": { + "light": "media/light/github.svg", + "dark": "media/dark/github.svg" + } + }, { "command": "codeQLDatabases.chooseDatabaseLgtm", "title": "Download from LGTM", @@ -428,6 +438,10 @@ "command": "codeQL.chooseDatabaseInternet", "title": "CodeQL: Download Database" }, + { + "command": "codeQL.chooseDatabaseGithub", + "title": "CodeQL: Download Database from GitHub" + }, { "command": "codeQL.chooseDatabaseLgtm", "title": "CodeQL: Download Database from LGTM" @@ -604,6 +618,11 @@ "when": "view == codeQLDatabases", "group": "navigation" }, + { + "command": "codeQLDatabases.chooseDatabaseGithub", + "when": "config.codeQL.canary && view == codeQLDatabases", + "group": "navigation" + }, { "command": "codeQLDatabases.chooseDatabaseLgtm", "when": "view == codeQLDatabases", @@ -829,6 +848,10 @@ "command": "codeQL.viewCfg", "when": "resourceScheme == codeql-zip-archive && config.codeQL.canary" }, + { + "command": "codeQL.chooseDatabaseGithub", + "when": "config.codeQL.canary" + }, { "command": "codeQLDatabases.setCurrentDatabase", "when": "false" @@ -873,6 +896,10 @@ "command": "codeQLDatabases.chooseDatabaseInternet", "when": "false" }, + { + "command": "codeQLDatabases.chooseDatabaseGithub", + "when": "false" + }, { "command": "codeQLDatabases.chooseDatabaseLgtm", "when": "false" diff --git a/extensions/ql-vscode/src/databaseFetcher.ts b/extensions/ql-vscode/src/databaseFetcher.ts index 6f64b0aa7ce..a3c9aa04c44 100644 --- a/extensions/ql-vscode/src/databaseFetcher.ts +++ b/extensions/ql-vscode/src/databaseFetcher.ts @@ -21,6 +21,8 @@ import { } from './commandRunner'; import { logger } from './logging'; import { tmpDir } from './helpers'; +import { Credentials } from './authentication'; +import { REPO_REGEX } from './pure/helpers-pure'; /** * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file. @@ -46,6 +48,7 @@ export async function promptImportInternetDatabase( const item = await databaseArchiveFetcher( databaseUrl, + {}, databaseManager, storagePath, progress, @@ -61,6 +64,79 @@ export async function promptImportInternetDatabase( } +/** + * Prompts a user to fetch a database from GitHub. + * User enters a GitHub repository and then the user is asked which language + * to download (if there is more than one) + * + * @param databaseManager the DatabaseManager + * @param storagePath where to store the unzipped database. + */ +export async function promptImportGithubDatabase( + databaseManager: DatabaseManager, + storagePath: string, + credentials: Credentials, + progress: ProgressCallback, + token: CancellationToken, + cli?: CodeQLCliServer +): Promise { + progress({ + message: 'Choose repository', + step: 1, + maxStep: 2 + }); + const githubRepo = await window.showInputBox({ + title: 'Enter a GitHub repository in the format / (e.g. github/codeql)', + placeHolder: '/', + ignoreFocusOut: true, + }); + if (!githubRepo) { + return; + } + + if (!REPO_REGEX.test(githubRepo)) { + throw new Error(`Invalid GitHub repository: ${githubRepo}`); + } + + const databaseUrl = await convertGithubNwoToDatabaseUrl(githubRepo, credentials, progress); + if (!databaseUrl) { + return; + } + + const octokit = await credentials.getOctokit(); + /** + * The 'token' property of the token object returned by `octokit.auth()`. + * The object is undocumented, but looks something like this: + * { + * token: 'xxxx', + * tokenType: 'oauth', + * type: 'token', + * } + * We only need the actual token string. + */ + const octokitToken = (await octokit.auth() as { token: string })?.token; + if (!octokitToken) { + // Just print a generic error message for now. Ideally we could show more debugging info, like the + // octokit object, but that would expose a user token. + throw new Error('Unable to get GitHub token.'); + } + const item = await databaseArchiveFetcher( + databaseUrl, + { 'Accept': 'application/zip', 'Authorization': `Bearer ${octokitToken}` }, + databaseManager, + storagePath, + progress, + token, + cli + ); + if (item) { + await commands.executeCommand('codeQLDatabases.focus'); + void showAndLogInformationMessage('Database downloaded and imported successfully.'); + return item; + } + return; +} + /** * Prompts a user to fetch a database from lgtm. * User enters a project url and then the user is asked which language @@ -94,6 +170,7 @@ export async function promptImportLgtmDatabase( if (databaseUrl) { const item = await databaseArchiveFetcher( databaseUrl, + {}, databaseManager, storagePath, progress, @@ -140,6 +217,7 @@ export async function importArchiveDatabase( try { const item = await databaseArchiveFetcher( databaseUrl, + {}, databaseManager, storagePath, progress, @@ -166,6 +244,7 @@ export async function importArchiveDatabase( * or in the local filesystem. * * @param databaseUrl URL from which to grab the database + * @param requestHeaders Headers to send with the request * @param databaseManager the DatabaseManager * @param storagePath where to store the unzipped database. * @param progress callback to send progress messages to @@ -173,6 +252,7 @@ export async function importArchiveDatabase( */ async function databaseArchiveFetcher( databaseUrl: string, + requestHeaders: { [key: string]: string }, databaseManager: DatabaseManager, storagePath: string, progress: ProgressCallback, @@ -193,7 +273,7 @@ async function databaseArchiveFetcher( if (isFile(databaseUrl)) { await readAndUnzip(databaseUrl, unzipPath, cli, progress); } else { - await fetchAndUnzip(databaseUrl, unzipPath, cli, progress); + await fetchAndUnzip(databaseUrl, requestHeaders, unzipPath, cli, progress); } progress({ @@ -292,6 +372,7 @@ async function readAndUnzip( async function fetchAndUnzip( databaseUrl: string, + requestHeaders: { [key: string]: string }, unzipPath: string, cli?: CodeQLCliServer, progress?: ProgressCallback @@ -310,7 +391,10 @@ async function fetchAndUnzip( step: 1, }); - const response = await checkForFailingResponse(await fetch(databaseUrl), 'Error downloading database'); + const response = await checkForFailingResponse( + await fetch(databaseUrl, { headers: requestHeaders }), + 'Error downloading database' + ); const archiveFileStream = fs.createWriteStream(archivePath); const contentLength = response.headers.get('content-length'); @@ -381,6 +465,37 @@ export async function findDirWithFile( return; } +export async function convertGithubNwoToDatabaseUrl( + githubRepo: string, + credentials: Credentials, + progress: ProgressCallback): Promise { + try { + // TODO: In future, we could accept GitHub URLs in addition to NWOs. + // Similar to "looksLikeLgtmUrl". + if (!REPO_REGEX.test(githubRepo)) { + throw new Error('Invalid repository format. Must be in the format / (e.g. github/codeql)'); + } + + const [owner, repo] = githubRepo.split('/'); + + const octokit = await credentials.getOctokit(); + const response = await octokit.request('GET /repos/:owner/:repo/code-scanning/codeql/databases', { owner, repo }); + + const languages = response.data.map((db: any) => db.language); + + const language = await promptForLanguage(languages, progress); + if (!language) { + return; + } + + return `https://api.github.com/repos/${owner}/${repo}/code-scanning/codeql/databases/${language}`; + + } catch (e) { + void logger.log(`Error: ${e.message}`); + throw new Error(`Unable to get database for '${githubRepo}'`); + } +} + /** * The URL pattern is https://lgtm.com/projects/{provider}/{org}/{name}/{irrelevant-subpages}. * There are several possibilities for the provider: in addition to GitHub.com (g), @@ -506,7 +621,7 @@ async function promptForLanguage( maxStep: 2 }); if (!languages.length) { - return; + throw new Error('No databases found'); } if (languages.length === 1) { return languages[0]; diff --git a/extensions/ql-vscode/src/databases-ui.ts b/extensions/ql-vscode/src/databases-ui.ts index 7c4aeecd020..ee81b568f8c 100644 --- a/extensions/ql-vscode/src/databases-ui.ts +++ b/extensions/ql-vscode/src/databases-ui.ts @@ -33,11 +33,13 @@ import * as qsClient from './queryserver-client'; import { upgradeDatabaseExplicit } from './upgrades'; import { importArchiveDatabase, + promptImportGithubDatabase, promptImportInternetDatabase, promptImportLgtmDatabase, } from './databaseFetcher'; import { CancellationToken } from 'vscode'; import { asyncFilter } from './pure/helpers-pure'; +import { Credentials } from './authentication'; type ThemableIconPath = { light: string; dark: string } | string; @@ -219,7 +221,8 @@ export class DatabaseUI extends DisposableObject { private databaseManager: DatabaseManager, private readonly queryServer: qsClient.QueryServerClient | undefined, private readonly storagePath: string, - readonly extensionPath: string + readonly extensionPath: string, + private readonly getCredentials: () => Promise ) { super(); @@ -291,6 +294,20 @@ export class DatabaseUI extends DisposableObject { } ) ); + this.push( + commandRunnerWithProgress( + 'codeQLDatabases.chooseDatabaseGithub', + async ( + progress: ProgressCallback, + token: CancellationToken + ) => { + const credentials = await this.getCredentials(); + await this.handleChooseDatabaseGithub(credentials, progress, token); + }, + { + title: 'Adding database from GitHub', + }) + ); this.push( commandRunnerWithProgress( 'codeQLDatabases.chooseDatabaseLgtm', @@ -462,6 +479,21 @@ export class DatabaseUI extends DisposableObject { ); }; + handleChooseDatabaseGithub = async ( + credentials: Credentials, + progress: ProgressCallback, + token: CancellationToken + ): Promise => { + return await promptImportGithubDatabase( + this.databaseManager, + this.storagePath, + credentials, + progress, + token, + this.queryServer?.cliServer + ); + }; + handleChooseDatabaseLgtm = async ( progress: ProgressCallback, token: CancellationToken diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts index 6626b3f7618..39cca56ca9e 100644 --- a/extensions/ql-vscode/src/extension.ts +++ b/extensions/ql-vscode/src/extension.ts @@ -433,7 +433,8 @@ async function activateWithInstalledDistribution( dbm, qs, getContextStoragePath(ctx), - ctx.extensionPath + ctx.extensionPath, + () => Credentials.initialize(ctx), ); databaseUI.init(); ctx.subscriptions.push(databaseUI); @@ -931,6 +932,18 @@ async function activateWithInstalledDistribution( title: 'Choose a Database from an Archive' }) ); + ctx.subscriptions.push( + commandRunnerWithProgress('codeQL.chooseDatabaseGithub', async ( + progress: ProgressCallback, + token: CancellationToken + ) => { + const credentials = await Credentials.initialize(ctx); + await databaseUI.handleChooseDatabaseGithub(credentials, progress, token); + }, + { + title: 'Adding database from GitHub', + }) + ); ctx.subscriptions.push( commandRunnerWithProgress('codeQL.chooseDatabaseLgtm', ( progress: ProgressCallback, diff --git a/extensions/ql-vscode/src/pure/helpers-pure.ts b/extensions/ql-vscode/src/pure/helpers-pure.ts index 3940344b658..1c98b59e49f 100644 --- a/extensions/ql-vscode/src/pure/helpers-pure.ts +++ b/extensions/ql-vscode/src/pure/helpers-pure.ts @@ -35,3 +35,10 @@ export const ONE_DAY_IN_MS = 24 * 60 * 60 * 1000; export const ONE_HOUR_IN_MS = 1000 * 60 * 60; export const TWO_HOURS_IN_MS = 1000 * 60 * 60 * 2; export const THREE_HOURS_IN_MS = 1000 * 60 * 60 * 3; + +/** + * This regex matches strings of the form `owner/repo` where: + * - `owner` is made up of alphanumeric characters or single hyphens, starting and ending in an alphanumeric character + * - `repo` is made up of alphanumeric characters, hyphens, or underscores + */ +export const REPO_REGEX = /^(?:[a-zA-Z0-9]+-)*[a-zA-Z0-9]+\/[a-zA-Z0-9-_]+$/; diff --git a/extensions/ql-vscode/src/remote-queries/run-remote-query.ts b/extensions/ql-vscode/src/remote-queries/run-remote-query.ts index d59c0fdfa21..b509f68ce0d 100644 --- a/extensions/ql-vscode/src/remote-queries/run-remote-query.ts +++ b/extensions/ql-vscode/src/remote-queries/run-remote-query.ts @@ -22,6 +22,7 @@ import { OctokitResponse } from '@octokit/types/dist-types'; import { RemoteQuery } from './remote-query'; import { RemoteQuerySubmissionResult } from './remote-query-submission-result'; import { QueryMetadata } from '../pure/interface-types'; +import { REPO_REGEX } from '../pure/helpers-pure'; export interface QlPack { name: string; @@ -38,13 +39,6 @@ interface QueriesResponse { workflow_run_id: number } -/** - * This regex matches strings of the form `owner/repo` where: - * - `owner` is made up of alphanumeric characters or single hyphens, starting and ending in an alphanumeric character - * - `repo` is made up of alphanumeric characters, hyphens, or underscores - */ -const REPO_REGEX = /^(?:[a-zA-Z0-9]+-)*[a-zA-Z0-9]+\/[a-zA-Z0-9-_]+$/; - /** * Well-known names for the query pack used by the server. */ diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts b/extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts index 374bcb2f61f..1da22b6cdef 100644 --- a/extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts @@ -14,6 +14,9 @@ import { findDirWithFile, } from '../../databaseFetcher'; import { ProgressCallback } from '../../commandRunner'; +import * as pq from 'proxyquire'; + +const proxyquire = pq.noPreserveCache(); chai.use(chaiAsPromised); const expect = chai.expect; @@ -21,6 +24,132 @@ describe('databaseFetcher', function() { // These tests make API calls and may need extra time to complete. this.timeout(10000); + describe('convertGithubNwoToDatabaseUrl', () => { + let sandbox: sinon.SinonSandbox; + let quickPickSpy: sinon.SinonStub; + let progressSpy: ProgressCallback; + let mockRequest: sinon.SinonStub; + let mod: any; + + const credentials = getMockCredentials(0); + + beforeEach(() => { + sandbox = sinon.createSandbox(); + quickPickSpy = sandbox.stub(window, 'showQuickPick'); + progressSpy = sandbox.spy(); + mockRequest = sandbox.stub(); + mod = proxyquire('../../databaseFetcher', { + './authentication': { + Credentials: credentials, + }, + }); + }); + + afterEach(() => { + sandbox.restore(); + }); + + it('should convert a GitHub nwo to a database url', async () => { + // We can't make the real octokit request (since we need credentials), so we mock the response. + const mockApiResponse = { + data: [ + { + id: 1495869, + name: 'csharp-database', + language: 'csharp', + uploader: {}, + content_type: 'application/zip', + state: 'uploaded', + size: 55599715, + created_at: '2022-03-24T10:46:24Z', + updated_at: '2022-03-24T10:46:27Z', + url: 'https://api.github.com/repositories/143040428/code-scanning/codeql/databases/csharp', + }, + { + id: 1100671, + name: 'database.zip', + language: 'javascript', + uploader: {}, + content_type: 'application/zip', + state: 'uploaded', + size: 29294434, + created_at: '2022-03-01T16:00:04Z', + updated_at: '2022-03-01T16:00:06Z', + url: 'https://api.github.com/repositories/143040428/code-scanning/codeql/databases/javascript', + }, + { + id: 648738, + name: 'ql-database', + language: 'ql', + uploader: {}, + content_type: 'application/json; charset=utf-8', + state: 'uploaded', + size: 39735500, + created_at: '2022-02-02T09:38:50Z', + updated_at: '2022-02-02T09:38:51Z', + url: 'https://api.github.com/repositories/143040428/code-scanning/codeql/databases/ql', + }, + ], + }; + mockRequest.resolves(mockApiResponse); + quickPickSpy.resolves('javascript'); + const githubRepo = 'github/codeql'; + const dbUrl = await mod.convertGithubNwoToDatabaseUrl( + githubRepo, + credentials, + progressSpy + ); + + expect(dbUrl).to.equal( + 'https://api.github.com/repos/github/codeql/code-scanning/codeql/databases/javascript' + ); + expect(quickPickSpy.firstCall.args[0]).to.deep.equal([ + 'csharp', + 'javascript', + 'ql', + ]); + }); + + // Repository doesn't exist, or the user has no access to the repository. + it('should fail on an invalid/inaccessible repository', async () => { + const mockApiResponse = { + data: { + message: 'Not Found', + }, + status: 404, + }; + mockRequest.resolves(mockApiResponse); + const githubRepo = 'foo/bar-not-real'; + await expect( + mod.convertGithubNwoToDatabaseUrl(githubRepo, credentials, progressSpy) + ).to.be.rejectedWith(/Unable to get database/); + expect(progressSpy).to.have.callCount(0); + }); + + // User has access to the repository, but there are no databases for any language. + it('should fail on a repository with no databases', async () => { + const mockApiResponse = { + data: [], + }; + + mockRequest.resolves(mockApiResponse); + const githubRepo = 'foo/bar-with-no-dbs'; + await expect( + mod.convertGithubNwoToDatabaseUrl(githubRepo, credentials, progressSpy) + ).to.be.rejectedWith(/Unable to get database/); + expect(progressSpy).to.have.been.calledOnce; + }); + + function getMockCredentials(response: any) { + mockRequest = sinon.stub().resolves(response); + return { + getOctokit: () => ({ + request: mockRequest, + }), + }; + } + }); + describe('convertLgtmUrlToDatabaseUrl', () => { let sandbox: sinon.SinonSandbox; let quickPickSpy: sinon.SinonStub; diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/databases-ui.test.ts b/extensions/ql-vscode/src/vscode-tests/no-workspace/databases-ui.test.ts index 685c291860f..656a57f9572 100644 --- a/extensions/ql-vscode/src/vscode-tests/no-workspace/databases-ui.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/databases-ui.test.ts @@ -8,6 +8,7 @@ import { Uri } from 'vscode'; import { DatabaseUI } from '../../databases-ui'; import { testDisposeHandler } from '../test-dispose-handler'; +import { Credentials } from '../../authentication'; describe('databases-ui', () => { describe('fixDbUri', () => { @@ -78,7 +79,8 @@ describe('databases-ui', () => { } as any, {} as any, storageDir, - storageDir + storageDir, + () => Promise.resolve({} as Credentials), ); await databaseUI.handleRemoveOrphanedDatabases();