From 00145bbfd4f1c638b34798f052e93848d0234bba Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Thu, 27 Oct 2022 12:24:18 +0200 Subject: [PATCH 1/2] Add some basic integration tests for MRVA This adds some basic integration tests for MRVA using the GitHub mock API server. It only does basic assertions and still needs to stub some things because it is quite hard to properly test things since VSCode does not expose an API to e.g. answer quick pick pop-ups. I'm not sure how useful these integration tests will actually be in practice, but they do at least ensure that we are able to successfully submit a variant analysis. --- .../variant-analysis.integration.test.ts | 142 ++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/variant-analysis.integration.test.ts diff --git a/extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/variant-analysis.integration.test.ts b/extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/variant-analysis.integration.test.ts new file mode 100644 index 00000000000..0ed667dfa4e --- /dev/null +++ b/extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/variant-analysis.integration.test.ts @@ -0,0 +1,142 @@ +import * as path from 'path'; + +import * as sinon from 'sinon'; + +import { commands, extensions, TextDocument, window, workspace } from 'vscode'; +import { setupServer } from 'msw/node'; +import * as Octokit from '@octokit/rest'; +import { retry } from '@octokit/plugin-retry'; + +import { CodeQLExtensionInterface } from '../../../extension'; +import { createRequestHandlers } from '../../../mocks/request-handlers'; +import * as config from '../../../config'; +import { Credentials } from '../../../authentication'; + +const server = setupServer(); + +before(() => server.listen()); + +afterEach(() => server.resetHandlers()); + +after(() => server.close()); + +async function loadScenario(scenarioName: string) { + const handlers = await createRequestHandlers(path.join(__dirname, '../../../../src/mocks/scenarios', scenarioName)); + + server.use(...handlers); +} + +async function showQlDocument(name: string): Promise { + const folderPath = workspace.workspaceFolders![0].uri.fsPath; + const documentPath = path.resolve(folderPath, name); + const document = await workspace.openTextDocument(documentPath); + await window.showTextDocument(document!); + return document; +} + +describe('Variant Analysis Integration', function() { + this.timeout(10_000); + + let sandbox: sinon.SinonSandbox; + let quickPickSpy: sinon.SinonStub; + let inputBoxSpy: sinon.SinonStub; + let executeCommandSpy: sinon.SinonStub; + let showErrorMessageSpy: sinon.SinonStub; + + beforeEach(async () => { + sandbox = sinon.createSandbox(); + + sandbox.stub(config, 'isCanary').returns(true); + sandbox.stub(config, 'isVariantAnalysisLiveResultsEnabled').returns(true); + + const mockCredentials = { + getOctokit: () => Promise.resolve(new Octokit.Octokit({ retry })), + } as unknown as Credentials; + sandbox.stub(Credentials, 'initialize').resolves(mockCredentials); + + await config.setRemoteControllerRepo('github/vscode-codeql'); + + quickPickSpy = sandbox.stub(window, 'showQuickPick').resolves(undefined); + inputBoxSpy = sandbox.stub(window, 'showInputBox').resolves(undefined); + + executeCommandSpy = sandbox.stub(commands, 'executeCommand').callThrough(); + showErrorMessageSpy = sandbox.stub(window, 'showErrorMessage').resolves(undefined); + + try { + await extensions.getExtension>('GitHub.vscode-codeql')!.activate(); + } catch (e) { + fail(e as Error); + } + }); + + afterEach(() => { + sandbox.restore(); + }); + + describe('Successful scenario', () => { + beforeEach(async () => { + await loadScenario('problem-query-success'); + }); + + it('opens the variant analysis view', async () => { + await showQlDocument('query.ql'); + + // Select a repository list + quickPickSpy.onFirstCall().resolves({ + useCustomRepo: true, + }); + // Enter a GitHub repository + inputBoxSpy.onFirstCall().resolves('github/codeql'); + // Select target language for your query + quickPickSpy.onSecondCall().resolves('javascript'); + + await commands.executeCommand('codeQL.runVariantAnalysis'); + + sinon.assert.calledWith(executeCommandSpy, 'codeQL.openVariantAnalysisView', 146); + }); + }); + + describe('Missing controller repo', () => { + beforeEach(async () => { + await loadScenario('missing-controller-repo'); + }); + + it('shows the error message', async () => { + await showQlDocument('query.ql'); + + // Select a repository list + quickPickSpy.onFirstCall().resolves({ + useCustomRepo: true, + }); + // Enter a GitHub repository + inputBoxSpy.onFirstCall().resolves('github/codeql'); + + await commands.executeCommand('codeQL.runVariantAnalysis'); + + sinon.assert.calledWith(showErrorMessageSpy, sinon.match('Controller repository "github/vscode-codeql" not found'), sinon.match.string); + }); + }); + + describe('Submission failure', () => { + beforeEach(async () => { + await loadScenario('submission-failure'); + }); + + it('shows the error message', async () => { + await showQlDocument('query.ql'); + + // Select a repository list + quickPickSpy.onFirstCall().resolves({ + useCustomRepo: true, + }); + // Enter a GitHub repository + inputBoxSpy.onFirstCall().resolves('github/codeql'); + // Select target language for your query + quickPickSpy.onSecondCall().resolves('javascript'); + + await commands.executeCommand('codeQL.runVariantAnalysis'); + + sinon.assert.calledWith(showErrorMessageSpy, sinon.match('No repositories could be queried.'), sinon.match.string); + }); + }); +}); From 72b335649c80581fceb0f8360d9011b943e0bc0e Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Fri, 28 Oct 2022 17:22:34 +0200 Subject: [PATCH 2/2] Use new mock API server and rename integration test --- ...t-analysis-submission-integration.test.ts} | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) rename extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/{variant-analysis.integration.test.ts => variant-analysis-submission-integration.test.ts} (85%) diff --git a/extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/variant-analysis.integration.test.ts b/extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/variant-analysis-submission-integration.test.ts similarity index 85% rename from extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/variant-analysis.integration.test.ts rename to extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/variant-analysis-submission-integration.test.ts index 0ed667dfa4e..fa5623ed957 100644 --- a/extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/variant-analysis.integration.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/variant-analysis-submission-integration.test.ts @@ -3,28 +3,18 @@ import * as path from 'path'; import * as sinon from 'sinon'; import { commands, extensions, TextDocument, window, workspace } from 'vscode'; -import { setupServer } from 'msw/node'; import * as Octokit from '@octokit/rest'; import { retry } from '@octokit/plugin-retry'; import { CodeQLExtensionInterface } from '../../../extension'; -import { createRequestHandlers } from '../../../mocks/request-handlers'; import * as config from '../../../config'; import { Credentials } from '../../../authentication'; +import { MockGitHubApiServer } from '../../../mocks/mock-gh-api-server'; -const server = setupServer(); - -before(() => server.listen()); - -afterEach(() => server.resetHandlers()); - -after(() => server.close()); - -async function loadScenario(scenarioName: string) { - const handlers = await createRequestHandlers(path.join(__dirname, '../../../../src/mocks/scenarios', scenarioName)); - - server.use(...handlers); -} +const mockServer = new MockGitHubApiServer(); +before(() => mockServer.startServer()); +afterEach(() => mockServer.unloadScenario()); +after(() => mockServer.stopServer()); async function showQlDocument(name: string): Promise { const folderPath = workspace.workspaceFolders![0].uri.fsPath; @@ -34,7 +24,7 @@ async function showQlDocument(name: string): Promise { return document; } -describe('Variant Analysis Integration', function() { +describe('Variant Analysis Submission Integration', function() { this.timeout(10_000); let sandbox: sinon.SinonSandbox; @@ -75,7 +65,7 @@ describe('Variant Analysis Integration', function() { describe('Successful scenario', () => { beforeEach(async () => { - await loadScenario('problem-query-success'); + await mockServer.loadScenario('problem-query-success'); }); it('opens the variant analysis view', async () => { @@ -98,7 +88,7 @@ describe('Variant Analysis Integration', function() { describe('Missing controller repo', () => { beforeEach(async () => { - await loadScenario('missing-controller-repo'); + await mockServer.loadScenario('missing-controller-repo'); }); it('shows the error message', async () => { @@ -119,7 +109,7 @@ describe('Variant Analysis Integration', function() { describe('Submission failure', () => { beforeEach(async () => { - await loadScenario('submission-failure'); + await mockServer.loadScenario('submission-failure'); }); it('shows the error message', async () => {