perf(express): add benchmark express (#2431)

Add performance test for express. Uses a [git submodule](https://git-scm.com/book/en/v2/Git-Tools-Submodules) to grap sources directly from https://github.com/expressjs/express

Co-authored-by: Nico Jansen <jansennico@gmail.com>

.github/workflows/performance.yml

@@ -1,22 +1,24 @@
 name: Performance
 
 on:
-  schedule:
-    - cron:  '0 12 * * *'
+  workflow_dispatch:
+    inputs:
+      PERF_TEST_GLOB_PATTERN:
+        description: 'Glob pattern'     
+        required: true
+        default: '*'
 
 jobs:
   performance_test:
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v1
-    - name: Use Node.js 12.x
-      uses: actions/setup-node@v1
-      with:
-        node-version: 12.x
     - name: Install dependencies
       run: npm install
     - name: Build packages
       run: npm run build
     - name: Run performance tests
       run: npm run perf
+      env:
+        PERF_TEST_GLOB_PATTERN: ${{ github.event.inputs.PERF_TEST_GLOB_PATTERN }}

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "perf/test/express"]
+	path = perf/test/express
+	url = git@github.com:expressjs/express.git

package.json

@@ -8,6 +8,7 @@
     "@types/istanbul": "^0.4.29",
     "@types/karma": "^5.0.0",
     "@types/lodash": "^4.14.110",
+    "@types/minimatch": "^3.0.3",
     "@types/mkdirp": "1.0.0",
     "@types/mocha": "^7.0.2",
     "@types/node": "^14.0.1",
@@ -33,6 +34,7 @@
     "json-schema-to-typescript": "~9.1.0",
     "lerna": "^3.10.7",
     "link-parent-bin": "~1.0.0",
+    "minimatch": "^3.0.4",
     "mocha": "^8.0.1",
     "nyc": "^15.0.0",
     "prettier": "2.0.5",

perf/README.md

@@ -2,7 +2,20 @@
 
 **Experimental**
 
-Here you can find our performance tests. Run them with `npm run perf` from the root (after you've build with `npm run build`).
+Inside the `perf/test` directory you'll find some performance tests for Stryker in different use cases.
+We're using [git submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules) to clone (some of them) directly
+from github.
+
+## Setup
+
+1. Clone the Stryker mono repo
+2. Build the code. (`npm run build`). Makes sure there are no build errors.
+3. Checkout git submodules. For example:
+    ```
+    git submodule init 
+    git submodule update
+    ```
+4. Run `npm run perf` from the root, or run `npm install && npm run test` in the `perf` dir.
 
 ## Add new
 
@@ -13,8 +26,3 @@ The package you add should have [local dependencies](https://github.com/nicojs/n
 the Stryker packages you need.
 
 The script that is run in your package is `stryker`, so that npm script should perform the actual performance test.
-
-## Daily build
-
-Performance tests are run for every daily build (once a day).
-Right now, performance is only measured. The build doesn't break on thresholds (but we might introduce something like that later).

perf/config/express/package.json

@@ -0,0 +1,10 @@
+{
+  "localDependencies": {
+    "@stryker-mutator/core": "../../../packages/core",
+    "@stryker-mutator/api": "../../../packages/api",
+    "@stryker-mutator/javascript-mutator": "../../../packages/javascript-mutator",
+    "@stryker-mutator/mocha-framework": "../../../packages/mocha-framework",
+    "@stryker-mutator/mocha-runner": "../../../packages/mocha-runner",
+    "@stryker-mutator/util": "../../../packages/util"
+  }
+}

perf/config/express/stryker.conf.json

@@ -0,0 +1,7 @@
+{
+  "$schema": "../../../packages/core/schema/stryker-schema.json",
+  "testRunner": "mocha",
+  "testFramework": "mocha",
+  "coverageAnalysis": "perTest",
+  "mutator": "javascript"
+}

perf/package.json

@@ -6,6 +6,8 @@
     "ts-node": "^8.0.2"
   },
   "scripts": {
+    "merge-config": "ts-node -p tasks/merge-config.ts",
+    "preinstall": "npm run merge-config",
     "postinstall": "node ../node_modules/lerna/cli.js bootstrap && npm run install-local-dependencies",
     "install-local-dependencies": "ts-node -p tasks/install-local-dependencies.ts",
     "test": "ts-node tasks/run-perf-tests.ts"

perf/tasks/merge-config.ts

@@ -0,0 +1,39 @@
+import { promises as fs } from 'fs';
+import path = require("path");
+
+const testRootDir = path.resolve(__dirname, '..', 'test');
+const configRootDir = path.resolve(__dirname, '..', 'config');
+
+mergeConfiguration().catch(error => {
+  console.error(error);
+  process.exitCode = 1;
+});
+
+async function mergeConfiguration() {
+  const testDirs = (await fs.readdir(testRootDir, { withFileTypes: true })).filter(testDir => testDir.isDirectory());
+  for await (const testDir of testDirs) {
+    const configOverrideDir = path.resolve(configRootDir, testDir.name);
+    try {
+      const configDir = await fs.stat(configOverrideDir);
+      if (configDir.isDirectory()) {
+        const overridePackageFileName = path.resolve(configOverrideDir, 'package.json');
+        const overrideStrykerConfigFileName = path.resolve(configOverrideDir, 'stryker.conf.json');
+        try {
+          const overrides = require(overridePackageFileName);
+          const original = require(path.resolve(testRootDir, testDir.name, 'package.json'));
+          await fs.writeFile(path.resolve(testRootDir, testDir.name, 'package.json'), JSON.stringify({ ...original, ...overrides }, null, 2))
+        } catch {
+          console.log(`Note: no overrides found at ${overridePackageFileName}`);
+        }
+        try {
+          await fs.copyFile(overrideStrykerConfigFileName, path.resolve(testRootDir, testDir.name, 'stryker.conf.json'));
+        } catch {
+          console.log(`Note: no stryker.conf.json file ${overrideStrykerConfigFileName}`);
+        }
+      }
+      console.log(`✅ Merged config for ${testDir.name}`)
+    } catch {
+      console.log(`Note: no config override directory found at ${configOverrideDir}`);
+    }
+  }
+}

perf/tasks/run-perf-tests.ts

@@ -3,6 +3,7 @@ import * as path from 'path';
 import * as execa from 'execa';
 import { Observable } from 'rxjs';
 import { tap, throttleTime } from 'rxjs/operators';
+import * as minimatch from 'minimatch';
 
 const testRootDir = path.resolve(__dirname, '..', 'test');
 
@@ -14,7 +15,14 @@ runPerfTests()
   });
 
 async function runPerfTests() {
-  const testDirs = fs.readdirSync(testRootDir);
+  const globPattern = process.env.PERF_TEST_GLOB_PATTERN || '*';
+  const testDirs = fs.readdirSync(testRootDir).filter(testDir => minimatch(testDir, globPattern));
+  if (testDirs.length) {
+    console.log(`Running performance tests on ${testDirs.join(', ')} (matched with glob pattern "${globPattern}")`)
+  } else {
+    console.warn(`No test files match glob expression ${globPattern}`);
+  }
+
   console.time('all tests');
   for (const testDir of testDirs) {
     await runTest(testDir);
@@ -24,19 +32,19 @@ async function runPerfTests() {
 
 async function runTest(testDir: string) {
   console.time(testDir);
-  await execNpm(['run', 'stryker'], testDir).pipe(
+  await npx(['stryker', 'run'], testDir).pipe(
     throttleTime(60000),
     tap(logMessage => console.timeLog(testDir, 'last log message: ', logMessage))
   ).toPromise();
   console.timeEnd(testDir);
 }
 
-function execNpm(args: string[], testDir: string): Observable<string> {
+function npx(args: string[], testDir: string): Observable<string> {
   const currentTestDir = path.resolve(testRootDir, testDir);
-  console.log(`Exec ${testDir} npm ${args.join(' ')}`);
+  console.log(`Exec ${testDir} npx ${args.join(' ')}`);
 
   return new Observable(observer => {
-    const testProcess = execa('npm', args, { timeout: 0, cwd: currentTestDir, stdio: 'pipe' });
+    const testProcess = execa('npx', args, { timeout: 0, cwd: currentTestDir, stdio: 'pipe' });
     let stderr = '';
     testProcess.stderr?.on('data', chunk => stderr += chunk.toString());
     testProcess.stdout?.on('data', chunk => observer.next(chunk.toString().trim()));