diff --git a/.mocha-reporter.js b/.mocha-reporter.js index 61f264b37..9a2f07da1 100644 --- a/.mocha-reporter.js +++ b/.mocha-reporter.js @@ -12,20 +12,22 @@ // //===----------------------------------------------------------------------===// -const BaseReporter = require("mocha/lib/reporters/base"); -const SpecReporter = require("mocha/lib/reporters/spec"); -const JsonReporter = require("mocha/lib/reporters/json"); +const mocha = require("mocha"); +const GHASummaryReporter = require("./dist/test/reporters/GitHubActionsSummaryReporter"); // Taking inspiration from https://github.com/stanleyhlng/mocha-multi-reporters/issues/108#issuecomment-2028773686 // since mocha-multi-reporters seems to have bugs with newer mocha versions -module.exports = class MultiReporter extends BaseReporter { +module.exports = class MultiReporter extends mocha.reporters.Base { constructor(runner, options) { super(runner, options); this.reporters = [ - new SpecReporter(runner, { + new mocha.reporters.Spec(runner, { reporterOption: options.reporterOption.specReporterOptions, }), - new JsonReporter(runner, { + new GHASummaryReporter(runner, { + reporterOption: options.reporterOption.githubActionsSummaryReporterOptions, + }), + new mocha.reporters.JSON(runner, { reporterOption: options.reporterOption.jsonReporterOptions, }), ]; diff --git a/.vscode-test.js b/.vscode-test.js index 0b4f7769c..3dd490709 100644 --- a/.vscode-test.js +++ b/.vscode-test.js @@ -23,7 +23,8 @@ const dataDir = process.env["VSCODE_DATA_DIR"]; // Check if we're debugging by looking at the process executable. Unfortunately, the VS Code debugger // doesn't seem to allow setting environment variables on a launched extension host. -const isDebugRun = !(process.env["_"] ?? "").endsWith("node_modules/.bin/vscode-test"); +const processPath = process.env["_"] ?? ""; +const isDebugRun = !isCIBuild && !processPath.endsWith("node_modules/.bin/vscode-test"); function log(/** @type {string} */ message) { if (!isDebugRun) { @@ -122,6 +123,9 @@ module.exports = defineConfig({ retries: 1, reporter: path.join(__dirname, ".mocha-reporter.js"), reporterOptions: { + githubActionsSummaryReporterOptions: { + title: "Integration Test Summary", + }, jsonReporterOptions: { output: path.join(__dirname, "test-results", "integration-tests.json"), }, @@ -155,6 +159,9 @@ module.exports = defineConfig({ retries: 1, reporter: path.join(__dirname, ".mocha-reporter.js"), reporterOptions: { + githubActionsSummaryReporterOptions: { + title: "Code Workspace Test Summary", + }, jsonReporterOptions: { output: path.join(__dirname, "test-results", "code-workspace-tests.json"), }, @@ -177,6 +184,9 @@ module.exports = defineConfig({ slow: 100, reporter: path.join(__dirname, ".mocha-reporter.js"), reporterOptions: { + githubActionsSummaryReporterOptions: { + title: "Unit Test Summary", + }, jsonReporterOptions: { output: path.join(__dirname, "test-results", "unit-tests.json"), }, diff --git a/package-lock.json b/package-lock.json index 17a7b78c1..777c6ef3d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,12 +19,14 @@ "zod": "^4.1.5" }, "devDependencies": { + "@actions/core": "^1.11.1", "@trivago/prettier-plugin-sort-imports": "^5.2.2", "@types/archiver": "^6.0.3", "@types/chai": "^4.3.19", "@types/chai-as-promised": "^7.1.8", "@types/chai-subset": "^1.3.6", "@types/decompress": "^4.2.7", + "@types/diff": "^7.0.2", "@types/lcov-parse": "^1.0.2", "@types/lodash.debounce": "^4.0.9", "@types/lodash.throttle": "^4.1.9", @@ -52,6 +54,7 @@ "chai-subset": "^1.6.0", "decompress": "^4.2.1", "del-cli": "^6.0.0", + "diff": "^8.0.2", "esbuild": "^0.25.9", "eslint": "^8.57.0", "eslint-config-prettier": "^10.1.8", @@ -93,6 +96,45 @@ "node": ">=0.10.0" } }, + "node_modules/@actions/core": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.11.1.tgz", + "integrity": "sha512-hXJCSrkwfA46Vd9Z3q4cpEpHB1rL5NG04+/rbqW9d3+CSvtB1tYe8UTpAlixa1vj0m/ULglfEK2UKxMGxCxv5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@actions/exec": "^1.1.1", + "@actions/http-client": "^2.0.1" + } + }, + "node_modules/@actions/exec": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.1.1.tgz", + "integrity": "sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@actions/io": "^1.0.1" + } + }, + "node_modules/@actions/http-client": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.2.3.tgz", + "integrity": "sha512-mx8hyJi/hjFvbPokCg4uRd4ZX78t+YyRPtnKWwIl+RzNaVuFpQHfmlGVfsKEJN8LwTCvL+DfVgAM04XaHkm6bA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tunnel": "^0.0.6", + "undici": "^5.25.4" + } + }, + "node_modules/@actions/io": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.1.3.tgz", + "integrity": "sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q==", + "dev": true, + "license": "MIT" + }, "node_modules/@azu/format-text": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@azu/format-text/-/format-text-1.0.2.tgz", @@ -953,6 +995,16 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@fastify/busboy": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", + "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, "node_modules/@gar/promisify": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz", @@ -2106,6 +2158,13 @@ "@types/node": "*" } }, + "node_modules/@types/diff": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@types/diff/-/diff-7.0.2.tgz", + "integrity": "sha512-JSWRMozjFKsGlEjiiKajUjIJVKuKdE3oVy2DNtK+fUo8q82nhFZ2CPQwicAIkXrofahDXrWJ7mjelvZphMS98Q==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", @@ -4945,9 +5004,9 @@ } }, "node_modules/diff": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz", - "integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==", + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.2.tgz", + "integrity": "sha512-sSuxWU5j5SR9QQji/o2qMvqRNYRDOcBTgsJ/DeCf4iSN4gW+gNMXM7wFIP+fdXZxoNiAnHUTGjCr+TSWXdRDKg==", "dev": true, "license": "BSD-3-Clause", "engines": { @@ -7947,6 +8006,16 @@ "node": ">=12" } }, + "node_modules/mocha/node_modules/diff": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz", + "integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/mocha/node_modules/glob": { "version": "10.4.5", "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", @@ -9707,6 +9776,16 @@ "sinon": ">=4.0.0" } }, + "node_modules/sinon/node_modules/diff": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz", + "integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/slash": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/slash/-/slash-5.1.0.tgz", @@ -10792,6 +10871,19 @@ "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==", "dev": true }, + "node_modules/undici": { + "version": "5.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.29.0.tgz", + "integrity": "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@fastify/busboy": "^2.0.0" + }, + "engines": { + "node": ">=14.0" + } + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", @@ -11393,6 +11485,41 @@ "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==", "dev": true }, + "@actions/core": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.11.1.tgz", + "integrity": "sha512-hXJCSrkwfA46Vd9Z3q4cpEpHB1rL5NG04+/rbqW9d3+CSvtB1tYe8UTpAlixa1vj0m/ULglfEK2UKxMGxCxv5A==", + "dev": true, + "requires": { + "@actions/exec": "^1.1.1", + "@actions/http-client": "^2.0.1" + } + }, + "@actions/exec": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.1.1.tgz", + "integrity": "sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==", + "dev": true, + "requires": { + "@actions/io": "^1.0.1" + } + }, + "@actions/http-client": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.2.3.tgz", + "integrity": "sha512-mx8hyJi/hjFvbPokCg4uRd4ZX78t+YyRPtnKWwIl+RzNaVuFpQHfmlGVfsKEJN8LwTCvL+DfVgAM04XaHkm6bA==", + "dev": true, + "requires": { + "tunnel": "^0.0.6", + "undici": "^5.25.4" + } + }, + "@actions/io": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.1.3.tgz", + "integrity": "sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q==", + "dev": true + }, "@azu/format-text": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@azu/format-text/-/format-text-1.0.2.tgz", @@ -11917,6 +12044,12 @@ "integrity": "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==", "dev": true }, + "@fastify/busboy": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", + "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", + "dev": true + }, "@gar/promisify": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz", @@ -12802,6 +12935,12 @@ "@types/node": "*" } }, + "@types/diff": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@types/diff/-/diff-7.0.2.tgz", + "integrity": "sha512-JSWRMozjFKsGlEjiiKajUjIJVKuKdE3oVy2DNtK+fUo8q82nhFZ2CPQwicAIkXrofahDXrWJ7mjelvZphMS98Q==", + "dev": true + }, "@types/istanbul-lib-coverage": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", @@ -14861,9 +15000,9 @@ "optional": true }, "diff": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz", - "integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==", + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.2.tgz", + "integrity": "sha512-sSuxWU5j5SR9QQji/o2qMvqRNYRDOcBTgsJ/DeCf4iSN4gW+gNMXM7wFIP+fdXZxoNiAnHUTGjCr+TSWXdRDKg==", "dev": true }, "doctrine": { @@ -17057,6 +17196,12 @@ "wrap-ansi": "^7.0.0" } }, + "diff": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz", + "integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==", + "dev": true + }, "glob": { "version": "10.4.5", "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", @@ -18270,6 +18415,14 @@ "@sinonjs/samsam": "^8.0.1", "diff": "^7.0.0", "supports-color": "^7.2.0" + }, + "dependencies": { + "diff": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz", + "integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==", + "dev": true + } } }, "sinon-chai": { @@ -19062,6 +19215,15 @@ "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==", "dev": true }, + "undici": { + "version": "5.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.29.0.tgz", + "integrity": "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==", + "dev": true, + "requires": { + "@fastify/busboy": "^2.0.0" + } + }, "undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", diff --git a/package.json b/package.json index da953def7..b6deb20c4 100644 --- a/package.json +++ b/package.json @@ -1986,12 +1986,14 @@ "**/*": "prettier --write --ignore-unknown" }, "devDependencies": { + "@actions/core": "^1.11.1", "@trivago/prettier-plugin-sort-imports": "^5.2.2", "@types/archiver": "^6.0.3", "@types/chai": "^4.3.19", "@types/chai-as-promised": "^7.1.8", "@types/chai-subset": "^1.3.6", "@types/decompress": "^4.2.7", + "@types/diff": "^7.0.2", "@types/lcov-parse": "^1.0.2", "@types/lodash.debounce": "^4.0.9", "@types/lodash.throttle": "^4.1.9", @@ -2019,6 +2021,7 @@ "chai-subset": "^1.6.0", "decompress": "^4.2.1", "del-cli": "^6.0.0", + "diff": "^8.0.2", "esbuild": "^0.25.9", "eslint": "^8.57.0", "eslint-config-prettier": "^10.1.8", diff --git a/test/integration-tests/ExtensionActivation.test.ts b/test/integration-tests/ExtensionActivation.test.ts index dd9b69d6b..14ccfc21f 100644 --- a/test/integration-tests/ExtensionActivation.test.ts +++ b/test/integration-tests/ExtensionActivation.test.ts @@ -27,7 +27,7 @@ import { deactivateExtension, } from "./utilities/testutilities"; -suite("Extension Activation/Deactivation Tests", () => { +tag("medium").suite("Extension Activation/Deactivation Tests", () => { suite("Extension Activation", () => { afterEach(async () => { await deactivateExtension(); @@ -103,7 +103,7 @@ suite("Extension Activation/Deactivation Tests", () => { }); }); - tag("medium").suite("Activates for cmake projects", function () { + suite("Activates for cmake projects", function () { let workspaceContext: WorkspaceContext; activateExtensionForTest({ diff --git a/test/integration-tests/commands/captureDiagnostics.test.ts b/test/integration-tests/commands/captureDiagnostics.test.ts index 2f665338b..dbc1e7408 100644 --- a/test/integration-tests/commands/captureDiagnostics.test.ts +++ b/test/integration-tests/commands/captureDiagnostics.test.ts @@ -30,7 +30,7 @@ import { updateSettings, } from "../utilities/testutilities"; -suite("captureDiagnostics Test Suite", () => { +tag("medium").suite("captureDiagnostics Test Suite", () => { let workspaceContext: WorkspaceContext; const mockWindow = mockGlobalObject(vscode, "window"); diff --git a/test/integration-tests/tasks/SwiftPluginTaskProvider.test.ts b/test/integration-tests/tasks/SwiftPluginTaskProvider.test.ts index 3141e90a5..743f52c39 100644 --- a/test/integration-tests/tasks/SwiftPluginTaskProvider.test.ts +++ b/test/integration-tests/tasks/SwiftPluginTaskProvider.test.ts @@ -22,6 +22,7 @@ import { SwiftExecution } from "@src/tasks/SwiftExecution"; import { SwiftPluginTaskProvider } from "@src/tasks/SwiftPluginTaskProvider"; import { SwiftTask } from "@src/tasks/SwiftTaskProvider"; +import { tag } from "../../tags"; import { cleanOutput, executeTaskAndWaitForResult, @@ -34,7 +35,7 @@ import { updateSettings, } from "../utilities/testutilities"; -suite("SwiftPluginTaskProvider Test Suite", function () { +tag("medium").suite("SwiftPluginTaskProvider Test Suite", function () { let workspaceContext: WorkspaceContext; let folderContext: FolderContext; diff --git a/test/integration-tests/tasks/SwiftTaskProvider.test.ts b/test/integration-tests/tasks/SwiftTaskProvider.test.ts index 7315fdd92..0fa652de4 100644 --- a/test/integration-tests/tasks/SwiftTaskProvider.test.ts +++ b/test/integration-tests/tasks/SwiftTaskProvider.test.ts @@ -102,18 +102,18 @@ suite("SwiftTaskProvider Test Suite", () => { }); suite("includes build all task from extension", () => { - let task: vscode.Task | undefined; - - setup(async () => { + async function getBuildAllTask(): Promise { const tasks = await vscode.tasks.fetchTasks({ type: "swift" }); - task = tasks.find(t => t.name === "Build All (defaultPackage)"); - }); + return tasks.find(t => t.name === "Build All (defaultPackage)"); + } test("provided", async () => { + const task = await getBuildAllTask(); expect(task?.detail).to.include("swift build --build-tests"); }); tag("large").test("executes", async () => { + const task = await getBuildAllTask(); assert(task); const exitPromise = waitForEndTaskProcess(task); await vscode.tasks.executeTask(task); @@ -123,23 +123,23 @@ suite("SwiftTaskProvider Test Suite", () => { }); suite("includes build all task from tasks.json", () => { - let task: vscode.Task | undefined; - - setup(async () => { + async function getBuildAllTask(): Promise { const tasks = await vscode.tasks.fetchTasks({ type: "swift" }); - task = tasks.find( + return tasks.find( t => t.name === "swift: Build All from " + (vscode.workspace.workspaceFile ? "code workspace" : "tasks.json") ); - }); + } test("provided", async () => { + const task = await getBuildAllTask(); expect(task?.detail).to.include("swift build --show-bin-path"); }); test("executes", async () => { + const task = await getBuildAllTask(); assert(task); const exitPromise = waitForEndTaskProcess(task); await vscode.tasks.executeTask(task); diff --git a/test/integration-tests/tasks/TaskManager.test.ts b/test/integration-tests/tasks/TaskManager.test.ts index 45de5cf47..34624a0fb 100644 --- a/test/integration-tests/tasks/TaskManager.test.ts +++ b/test/integration-tests/tasks/TaskManager.test.ts @@ -17,9 +17,10 @@ import * as vscode from "vscode"; import { WorkspaceContext } from "@src/WorkspaceContext"; import { TaskManager } from "@src/tasks/TaskManager"; +import { tag } from "../../tags"; import { activateExtensionForSuite } from "../utilities/testutilities"; -suite("TaskManager Test Suite", () => { +tag("medium").suite("TaskManager Test Suite", () => { let workspaceContext: WorkspaceContext; let taskManager: TaskManager; diff --git a/test/integration-tests/tasks/TaskQueue.test.ts b/test/integration-tests/tasks/TaskQueue.test.ts index 9a84c7330..1f26b9376 100644 --- a/test/integration-tests/tasks/TaskQueue.test.ts +++ b/test/integration-tests/tasks/TaskQueue.test.ts @@ -21,7 +21,7 @@ import { testAssetPath } from "../../fixtures"; import { tag } from "../../tags"; import { activateExtensionForSuite, findWorkspaceFolder } from "../utilities/testutilities"; -suite("TaskQueue Test Suite", () => { +tag("medium").suite("TaskQueue Test Suite", () => { let workspaceContext: WorkspaceContext; let taskQueue: TaskQueue; @@ -131,7 +131,7 @@ suite("TaskQueue Test Suite", () => { // Queue two tasks. The first one taking longer than the second. If they // are queued correctly the first will still finish before the second - tag("medium").test("Test execution order", async () => { + test("Test execution order", async () => { const sleepScript = testAssetPath("sleep.sh"); const results: (number | undefined)[] = []; const task1 = new vscode.Task( diff --git a/test/integration-tests/utilities/testutilities.ts b/test/integration-tests/utilities/testutilities.ts index 6636e42a4..ffac1a893 100644 --- a/test/integration-tests/utilities/testutilities.ts +++ b/test/integration-tests/utilities/testutilities.ts @@ -64,7 +64,10 @@ const extensionBootstrapper = (() => { let workspaceContext: WorkspaceContext | undefined; let autoTeardown: void | (() => Promise); let restoreSettings: (() => Promise) | undefined; - before(async function () { + before("Activate Swift Extension", async function () { + // Allow enough time for the extension to activate + this.timeout(120_000); + // Make sure that CodeLLDB is installed for debugging related tests if (!vscode.extensions.getExtension("vadimcn.vscode-lldb")) { await vscode.commands.executeCommand( @@ -141,7 +144,10 @@ const extensionBootstrapper = (() => { } }); - after(async function () { + after("Deactivate Swift Extension", async function () { + // Allow enough time for the extension to deactivate + this.timeout(60_000); + let userTeardownError: unknown | undefined; try { // First run the users supplied teardown, then await the autoTeardown if it exists. diff --git a/test/reporters/GitHubActionsSummaryReporter.ts b/test/reporters/GitHubActionsSummaryReporter.ts new file mode 100644 index 000000000..91ebba795 --- /dev/null +++ b/test/reporters/GitHubActionsSummaryReporter.ts @@ -0,0 +1,179 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the VS Code Swift open source project +// +// Copyright (c) 2025 the VS Code Swift project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of VS Code Swift project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// +import { diffLines } from "diff"; +import * as fs from "fs"; +import * as mocha from "mocha"; + +const SUMMARY_ENV_VAR = "GITHUB_STEP_SUMMARY"; + +interface AssertionError extends Error { + showDiff?: boolean; + actual: string; + expected: string; +} + +function isAssertionError(err: Error): err is AssertionError { + return typeof (err as any).actual === "string" && typeof (err as any).expected === "string"; +} + +module.exports = class GitHubActionsSummaryReporter extends mocha.reporters.Base { + private _summaryFilePath: string | null | undefined; + get summaryFilePath(): string | null | undefined { + if (this._summaryFilePath !== undefined) { + return this._summaryFilePath; + } + + const summaryPath = process.env[SUMMARY_ENV_VAR]; + if (!summaryPath) { + this._summaryFilePath = null; + return null; + } + + try { + fs.accessSync(summaryPath, fs.constants.R_OK | fs.constants.W_OK); + } catch { + this._summaryFilePath = null; + return null; + } + + this._summaryFilePath = summaryPath; + return summaryPath; + } + + constructor(runner: Mocha.Runner, options: any) { + super(runner, options); + + const EVENT_RUN_END = mocha.Runner.constants.EVENT_RUN_END; + runner.on(EVENT_RUN_END, () => { + const title = options.reporterOption.title ?? "Test Summary"; + this.appendSummary(createMarkdownSummary(title, this.stats, this.failures)); + }); + } + + // Appends to the summary file synchronously since mocha does not support + // asynchronous reporters. + appendSummary(summary: string) { + if (!this.summaryFilePath) { + return; + } + fs.appendFileSync(this.summaryFilePath, summary, { encoding: "utf8" }); + } +}; + +function fullTitle(test: Mocha.Test | Mocha.Suite): string { + if (test.parent && test.parent.title) { + return fullTitle(test.parent) + " | " + test.title; + } + return test.title; +} + +function generateErrorMessage(failure: Mocha.Test): string { + if (!failure.err) { + return "The test did not report what the error was."; + } + + const stackTraceFilter = mocha.utils.stackTraceFilter(); + if (isAssertionError(failure.err) && failure.err.showDiff) { + const { message, stack } = splitStackTrace(failure.err); + return ( + message + + eol() + + generateDiff(failure.err.actual, failure.err.expected) + + eol() + + eol() + + stackTraceFilter(stack) + ); + } + if (failure.err.stack) { + return stackTraceFilter(failure.err.stack); + } + return mocha.utils.stringify(failure.err); +} + +function splitStackTrace(error: Error): { message: string; stack: string } { + if (!error.stack) { + return { message: error.message, stack: "" }; + } + + const indexOfMessage = error.stack.lastIndexOf(error.message); + const endIndexOfMessage = indexOfMessage + error.message.length; + return { + message: error.stack.substring(0, endIndexOfMessage), + stack: error.stack.substring(endIndexOfMessage + 1), + }; +} + +function generateDiff(actual: string, expected: string) { + return [ + "🟩 expected 🟥 actual\n\n", + ...diffLines(expected, actual).map(part => { + if (part.added) { + return "🟩" + part.value; + } + if (part.removed) { + return "🟥" + part.value; + } + return part.value; + }), + ].join(""); +} + +function tag(tag: string, attributes: string[], content: string): string { + return `<${tag}${attributes.length > 0 ? ` ${attributes.join(" ")}` : ""}>${content}`; +} + +function details(summary: string, open: boolean, content: string): string { + return tag( + "details", + open ? ["open"] : [], + eol() + tag("summary", [], summary) + eol() + content + eol() + ); +} + +function list(lines: string[]): string { + return tag("ul", [], eol() + lines.map(line => tag("li", [], line)).join(eol()) + eol()); +} + +function eol(): string { + if (process.platform === "win32") { + return "\r\n"; + } + return "\n"; +} + +function createMarkdownSummary(title: string, stats: Mocha.Stats, failures: Mocha.Test[]): string { + const isFailedRun = stats.failures > 0; + let summary = tag("h3", [], "Summary") + eol(); + summary += list([ + ...(stats.passes > 0 ? [`✅ ${stats.passes} passing test(s)`] : []), + ...(stats.failures > 0 ? [`❌ ${stats.failures} failing test(s)`] : []), + ...(stats.pending > 0 ? [`⚠️ ${stats.pending} pending test(s)`] : []), + ]); + if (isFailedRun) { + summary += tag("h3", [], "Test Failures"); + summary += list( + failures.map(failure => { + const errorMessage = generateErrorMessage(failure); + return ( + eol() + + tag("h5", [], fullTitle(failure)) + + eol() + + tag("pre", [], eol() + errorMessage + eol()) + + eol() + ); + }) + ); + } + return details(`${isFailedRun ? "❌" : "✅"} ${title}`, isFailedRun, summary) + eol(); +}