test: Use Vitest to run unit and e2e tests (#872)

getsentry · Apr 4, 2024 · 7bc6ad0 · 7bc6ad0
1 parent 70032e4
commit 7bc6ad0
Show file tree

Hide file tree

Showing 18 changed files with 993 additions and 809 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -83,7 +83,7 @@ jobs:
           node-version-file: 'package.json'
       - run: yarn install
       - name: Run E2E Tests
-        timeout-minutes: 25
+        timeout-minutes: 30
         run: yarn e2e
 
   required_jobs_passed:

diff --git a/package.json b/package.json
@@ -40,7 +40,7 @@
     "prebuild": "yarn clean && node scripts/update-version.js",
     "build": "rollup --config rollup.config.mjs",
     "postbuild": "node scripts/check-exports.mjs",
-    "clean": "rimraf coverage esm main preload renderer index.* integrations.* ipc.* sentry-electron*.tgz",
+    "clean": "rimraf --glob coverage common esm main preload renderer index.* sentry-electron*.tgz .eslintcache",
     "prelint": "node scripts/update-version.js",
     "lint": "run-s lint:prettier lint:eslint",
     "lint:prettier": "prettier --check \"{src,test}/**/*.ts\"",
@@ -51,9 +51,9 @@
     "update-electron-versions": "electron-latest-versions --start 15 --beta > ./test/e2e/versions.json",
     "update-sdk-versions": "node ./scripts/update-sdk-versions.mjs",
     "pretest": "yarn build",
-    "test": "cross-env TS_NODE_PROJECT=tsconfig.test.json xvfb-maybe electron-mocha --require ts-node/register/transpile-only --timeout 12000 ./test/unit/**/*.ts",
-    "pree2e": "rimraf test/e2e/dist/**/node_modules/@sentry/** test/e2e/dist/**/yarn.lock test/e2e/dist/**/package-lock.json && node scripts/clean-cache.js && yarn build && npm pack",
-    "e2e": "cross-env TS_NODE_PROJECT=tsconfig.test.json xvfb-maybe mocha --exit --require ts-node/register/transpile-only --retries 3 ./test/e2e/*.ts"
+    "test": "vitest run --root=./test/unit",
+    "pree2e": "rimraf --glob test/e2e/dist/**/node_modules/@sentry/** test/e2e/dist/**/yarn.lock test/e2e/dist/**/package-lock.json && node scripts/clean-cache.js && yarn build && npm pack",
+    "e2e": "xvfb-maybe vitest run --root=./test/e2e"
   },
   "dependencies": {
     "@sentry/browser": "8.0.0-alpha.7",
@@ -69,36 +69,26 @@
     "@sentry-internal/eslint-config-sdk": "8.0.0-alpha.7",
     "@sentry-internal/typescript": "8.0.0-alpha.7",
     "@types/busboy": "^0.2.3",
-    "@types/chai": "^4.2.10",
-    "@types/chai-as-promised": "^7.1.5",
-    "@types/chai-subset": "^1.3.3",
     "@types/form-data": "^2.5.0",
     "@types/koa": "^2.0.52",
     "@types/koa-bodyparser": "^4.3.0",
-    "@types/mocha": "^9.0.0",
     "@types/tmp": "^0.2.2",
     "busboy": "^0.3.1",
-    "chai": "^4.3.7",
-    "chai-as-promised": "^7.1.1",
-    "chai-subset": "^1.6.0",
-    "cross-env": "^7.0.3",
     "electron": "25.3.0",
     "electron-latest-versions": "^0.2.0",
-    "electron-mocha": "^11.0.2",
     "eslint": "7.32.0",
     "extract-zip": "^2.0.1",
     "koa": "^2.14.1",
     "koa-bodyparser": "^4.3.0",
     "koa-tree-router": "^0.12.1",
     "latest-version": "^7.0.0",
-    "mocha": "^10.2.0",
     "npm-run-all": "^4.1.5",
     "prettier": "^2.8.4",
-    "rimraf": "^3.0.2",
+    "rimraf": "^5.0.5",
     "tmp": "^0.2.1",
-    "ts-node": "^10.9.1",
     "rollup": "^4.13.1",
     "typescript": "^4.9.5",
+    "vitest": "^1.4.0",
     "xvfb-maybe": "^0.2.1",
     "yaml": "^2.2.1"
   },

diff --git a/test/README.md b/test/README.md
@@ -34,9 +34,6 @@ versions will be tested.
 Each individual e2e test consists of a test recipe which is a self contained application in its own right. There are
 simple [functional test recipes](./e2e/test-apps/) and [example app recipes](../examples/).
 
-Recipes are run sequentially and the events submitted to the mock server are compared to the expected events using
-`chai-subset`.
-
 Each recipe should contain:
 
 - `recipe.yml` (details below)

diff --git a/test/e2e/context.ts b/test/e2e/context.ts
@@ -27,9 +27,9 @@ function getDeleteDirectories(appName: string): string[] {
 const log = createLogger('Test Context');
 const appLog = createLogger('App');
 
-if (!process.env.DEBUG) {
+if (!process.env.DEBUG && !process.env.GITHUB_ACTIONS) {
   // tslint:disable-next-line
-  console.log('You can enable DEBUG=true to get verbose output.');
+  console.log('You can enable DEBUG=true to get verbose output from the tests.');
 }
 
 /** A class to start and stop Electron apps for E2E tests. */

diff --git a/test/e2e/index.test.ts b/test/e2e/index.test.ts
@@ -1,28 +1,30 @@
-import { should, use } from 'chai';
-import chaiAsPromised from 'chai-as-promised';
-import chaiSubset from 'chai-subset';
 import { join } from 'path';
+import { afterAll, afterEach, beforeAll, beforeEach, describe, onTestFailed, test } from 'vitest';
 
 import { TestContext } from './context';
 import { downloadElectron } from './download';
 import { getCategorisedTestRecipes, getExampleRecipes } from './recipe';
 import { TestServer } from './server';
 import { clearTestLog, getElectronTestVersions, outputTestLog } from './utils';
 
-should();
-use(chaiAsPromised);
-use(chaiSubset);
-
 const distDir = join(__dirname, 'dist');
 
+function hookTestFailure() {
+  if (process.env.FAILURE_LOG) {
+    onTestFailed(() => {
+      outputTestLog();
+    });
+  }
+}
+
 describe('E2E Tests', () => {
   const testServer = new TestServer();
 
-  before(() => {
+  beforeAll(() => {
     testServer.start();
   });
 
-  after(async () => {
+  afterAll(async () => {
     await testServer.stop();
   });
 
@@ -31,59 +33,64 @@ describe('E2E Tests', () => {
       let testContext: TestContext | undefined;
       const electronPath = downloadElectron(electronVersion);
 
-      beforeEach(async function () {
-        this.timeout(60_000);
+      beforeEach(async () => {
         await electronPath;
         testServer.clearEvents();
         clearTestLog();
-      });
+      }, 60_000);
 
-      afterEach(async function () {
-        this.timeout(10_000);
+      afterEach(async () => {
         await testContext?.stop();
-
-        if (process.env.FAILURE_LOG && this.currentTest?.state === 'failed') {
-          outputTestLog();
-        }
-
         testContext = undefined;
-      });
+      }, 10_000);
 
       describe('Functional Test Recipes', () => {
         const categories = getCategorisedTestRecipes(electronVersion);
 
         for (const category of Object.keys(categories)) {
           describe(category, () => {
             for (const recipe of categories[category]) {
-              const fn = recipe.only ? it.only : it;
-
-              fn(recipe.description, async function () {
-                if (!recipe.shouldRun()) {
-                  this.skip();
-                }
-
-                const [appPath, appName] = await recipe.prepare(this, distDir);
-                testContext = new TestContext(await electronPath, electronVersion, appPath, appName);
-                await recipe.runTests(testContext, testServer);
-              });
+              const testFn = recipe.only ? test.only : test;
+
+              testFn(
+                recipe.description,
+                async (ctx) => {
+                  if (!recipe.shouldRun()) {
+                    ctx.skip();
+                  }
+
+                  hookTestFailure();
+
+                  const [appPath, appName] = await recipe.prepare(distDir);
+                  testContext = new TestContext(await electronPath, electronVersion, appPath, appName);
+                  await recipe.runTests(testContext, testServer);
+                },
+                recipe.timeout,
+              );
             }
           });
         }
       });
 
       describe('Example App Recipes', () => {
         for (const recipe of getExampleRecipes(electronVersion)) {
-          const fn = recipe.only ? it.only : it;
-
-          fn(recipe.description, async function () {
-            if (!recipe.shouldRun()) {
-              this.skip();
-            }
-
-            const [appPath, appName] = await recipe.prepare(this, distDir);
-            testContext = new TestContext(await electronPath, electronVersion, appPath, appName);
-            await recipe.runTests(testContext, testServer);
-          });
+          const testFn = recipe.only ? test.only : test;
+
+          testFn(
+            recipe.description,
+            async (ctx) => {
+              if (!recipe.shouldRun()) {
+                ctx.skip();
+              }
+
+              hookTestFailure();
+
+              const [appPath, appName] = await recipe.prepare(distDir);
+              testContext = new TestContext(await electronPath, electronVersion, appPath, appName);
+              await recipe.runTests(testContext, testServer);
+            },
+            recipe.timeout,
+          );
         }
       });
     });

diff --git a/test/e2e/recipe/index.ts b/test/e2e/recipe/index.ts
@@ -1,9 +1,9 @@
 import { Event } from '@sentry/types';
 import { parseSemver } from '@sentry/utils';
-import { expect } from 'chai';
 import { spawnSync } from 'child_process';
 import { mkdirSync, writeFileSync } from 'fs';
 import { dirname, join } from 'path';
+import { expect } from 'vitest';
 
 import { SDK_VERSION } from '../../../src/main/version';
 import { delay } from '../../helpers';
@@ -138,12 +138,13 @@ export class RecipeRunner {
     return pkg.name;
   }
 
-  public async prepare(context: Mocha.Context, testBasePath: string): Promise<[string, string]> {
-    log(`Preparing recipe '${this.description}'`);
-
+  public get timeout(): number {
     const timeout = (this._recipe.metadata.timeout || 30) * 1_000;
-    // macOS runs quite slowly in GitHub actions
-    context.timeout(process.platform === 'darwin' ? timeout * 2 : timeout);
+    return process.platform === 'darwin' ? timeout * 2 : timeout;
+  }
+
+  public async prepare(testBasePath: string): Promise<[string, string]> {
+    log(`Preparing recipe '${this.description}'`);
 
     let appPath = join(testBasePath, this._appName);
 
@@ -273,7 +274,7 @@ export class RecipeRunner {
       const isSession = eventIsSession(expectedEvent.data);
 
       log(`Comparing ${isSession ? 'session' : 'event'} ${i + 1} of ${expectedEvents.length}`);
-      expect(testServer.events).to.containSubset([expectedEvent]);
+      expect(testServer.events).containSubset([expectedEvent]);
     }
 
     log('Event comparisons passed!');

diff --git a/test/e2e/test-apps/other/window-titles/events.ts b/test/e2e/test-apps/other/window-titles/events.ts
@@ -1,5 +1,5 @@
 import { Event } from '@sentry/types';
-import { expect } from 'chai';
+import { expect } from 'vitest';
 
 import { TestServerEvent } from '../../../server';
 

diff --git a/test/e2e/vitest.config.mts b/test/e2e/vitest.config.mts
@@ -0,0 +1,11 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    include: ['./**/*.test.ts'],
+    retry: process.env.GITHUB_ACTIONS ? 3 : 0,
+    disableConsoleIntercept: true,
+    silent: false,
+    reporters: process.env.GITHUB_ACTIONS ? ['verbose', 'github-actions'] : ['verbose'],
+  },
+});
diff --git a/test/unit/electron-breadcrumbs.test.ts b/test/unit/electron-breadcrumbs.test.ts
@@ -1,12 +1,9 @@
-import { expect, should, use } from 'chai';
-import chaiAsPromised = require('chai-as-promised');
-import { normalizeOptions } from '../../src/main/integrations/electron-breadcrumbs';
+import { describe, expect, test } from 'vitest';
 
-should();
-use(chaiAsPromised);
+import { normalizeOptions } from '../../src/main/integrations/electron-breadcrumbs';
 
 describe('Electron Breadcrumbs', () => {
-  it('Normalize Options', () => {
+  test('Normalize Options', () => {
     const options = normalizeOptions({
       app: false,
       powerMonitor: true,

diff --git a/test/unit/getPath.test.ts b/test/unit/getPath.test.ts
@@ -1,9 +1,9 @@
-import { expect } from 'chai';
 import { spawnSync } from 'child_process';
 import { join } from 'path';
+import { describe, expect, test } from 'vitest';
 
 describe('app.getPath', () => {
-  it('not called before init', () => {
+  test('not called before init', () => {
     const result = spawnSync('yarn', ['start'], { cwd: join(__dirname, 'getPath-test-app') });
     // status is null on Windows in CI for some unknown reason
     if (process.platform !== 'win32') {