feat(typescript checker): group mutants to improve performance 🚀 (#3900)

Add grouping mutants to the `@stryker-mutator/typescript-checker` to improve performance.

The typescript checker is slow because mutants are checked for compile errors one by one. Each time a mutant is checked, TypeScript will do full type-checking. This is sped up by running multiple checkers in parallel, using typescript's `--watch` mode, and only compiling in memory. Yet still, this is very slow, as a rule of thumb: 10x slower than not enabling this checker.

This PR improves this by grouping mutants unrelated to each other in the same typescript compilation run. Any compilation errors that result from one of the mutants in a group can always be associated with one mutant. 

One downside to this approach is that a TypeScript compilation sometimes forgets to report some of the errors (although it is a design goal of TypeScript to report them all). For example microsoft/TypeScript#46272

This is why you disable this grouping behavior by adding:

```json
{
  "typeScriptChecker": {
    "prioritizePerformanceOverAccuracy": false
  }
}
```

docs/typescript-checker.md

@@ -30,7 +30,10 @@ You can configure the typescript checker in the `stryker.conf.js` (or `stryker.c
 ```json
 {
   "checkers": ["typescript"],
-  "tsconfigFile": "tsconfig.json"
+  "tsconfigFile": "tsconfig.json",
+  "typeScriptChecker": {
+    "prioritizePerformanceOverAccuracy": true
+  }
 }
 ```
 
@@ -52,6 +55,12 @@ _Note: the following compiler options are always overridden by @stryker-mutator/
 }
 ```
 
+### `typeScriptChecker.prioritizePerformanceOverAccuracy` [`boolean`]
+
+Default: `true`
+
+Sets the performance strategy for the typescript-checker. Defaults to `true` which the fastest strategy with the consequence of losing some accuracy. The accuracy that is lost comes down to having mutants with a status other than `CompileError` while they should have this status. This result in a report that may not be 100% accurate. Setting this option to `false` results in an accurate report but may take (way) longer.
+
 ## Peer dependencies
 
 The `@stryker-mutator/typescript-checker` package for `stryker` to enable `typescript` support. As such, you should make sure you have the correct versions of its dependencies installed:

e2e/test/typescript-project-references/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "typescript-project-references",
+  "version": "0.0.0",
+  "private": true,
+  "description": "A module to perform an integration test",
+  "main": "index.js",
+  "scripts": {
+    "clean": "rimraf dist",
+    "prebuild": "npm run clean",
+    "build": "tsc -b tsconfig.json",
+    "pretest:unit": "npm run build",
+    "test:unit": "mocha",
+    "pretest": "rimraf \"reports\" \"dist\" \"stryker.log\"",
+    "test": "stryker run",
+    "posttest": "mocha --no-config --no-package --timeout 0 verify/verify.js"
+  },
+  "mocha": {
+    "spec":  [
+      "test/**/*.js"
+    ]
+  },
+  "author": "",
+  "license": "ISC"
+}

e2e/test/typescript-project-references/src/core/index.ts

@@ -0,0 +1,5 @@
+import { count } from '../utils/math.js';
+
+export function countArrayLength(todo: any[]): number {
+  return count(todo);
+}

e2e/test/typescript-project-references/src/core/job.ts

@@ -0,0 +1,7 @@
+import { toUpperCase } from '../utils/text.js';
+
+export function start(): string {
+  const logText = "Starting job";
+  console.log(toUpperCase(logText));
+  return logText;
+}

e2e/test/typescript-project-references/src/core/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.settings",
+  "compilerOptions": {
+    "outDir": "../dist/src"
+  },
+  "references": [
+    { "path": "../utils" }
+  ]
+}

e2e/test/typescript-project-references/src/core/tsconfig.tsbuildinfo

@@ -0,0 +1 @@
+{"program":{"fileNames":["../../../../node_modules/typescript/lib/lib.d.ts","../../../../node_modules/typescript/lib/lib.es5.d.ts","../../../../node_modules/typescript/lib/lib.dom.d.ts","../../../../node_modules/typescript/lib/lib.webworker.importscripts.d.ts","../../../../node_modules/typescript/lib/lib.scripthost.d.ts","./index.ts","./job.ts"],"fileInfos":["2dc8c927c9c162a773c6bb3cdc4f3286c23f10eedc67414028f9cb5951610f60",{"version":"f20c05dbfe50a208301d2a1da37b9931bce0466eb5a1f4fe240971b4ecc82b67","affectsGlobalScope":true},{"version":"9b087de7268e4efc5f215347a62656663933d63c0b1d7b624913240367b999ea","affectsGlobalScope":true},{"version":"7fac8cb5fc820bc2a59ae11ef1c5b38d3832c6d0dfaec5acdb5569137d09a481","affectsGlobalScope":true},{"version":"097a57355ded99c68e6df1b738990448e0bf170e606707df5a7c0481ff2427cd","affectsGlobalScope":true},{"version":"c9b6bdd48b8bdb8d8e7690c7cc18897a494b6ab17dc58083dacfaf14b846ab4f","signature":"40b6409b8d0dced1f6c3964012b7a7c1cd50e24c3242095d1c8cfc6cabe8bd31"},{"version":"e4c28c497fe6cc6364b113c181c32ba58e70f02d824295e72b15d9570b403104","signature":"9be66c79f48b4876970daed5167e069d7f12f1a1ca616ecaa0ca8280946344ca"}],"options":{"composite":true,"declaration":true,"declarationMap":true,"module":1,"noUnusedLocals":true,"noUnusedParameters":true,"strict":true,"target":1},"referencedMap":[],"exportedModulesMap":[],"semanticDiagnosticsPerFile":[1,3,2,5,4],"changeFileSet":[6,7],"latestChangedDtsFile":"./job.d.ts"},"version":"4.8.4"}

e2e/test/typescript-project-references/src/utils/math.ts

@@ -0,0 +1,3 @@
+export function count(array: any[]) {
+  return array.length;
+}

e2e/test/typescript-project-references/src/utils/text.ts

@@ -0,0 +1,3 @@
+export function toUpperCase(text: string) {
+  return text.toUpperCase();
+}

e2e/test/typescript-project-references/src/utils/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends": "../../tsconfig.settings",
+  "compilerOptions": {
+    "outDir": "../dist/utils",
+  }
+}

e2e/test/typescript-project-references/src/utils/tsconfig.tsbuildinfo

@@ -0,0 +1 @@
+{"program":{"fileNames":["../../../../node_modules/typescript/lib/lib.d.ts","../../../../node_modules/typescript/lib/lib.es5.d.ts","../../../../node_modules/typescript/lib/lib.dom.d.ts","../../../../node_modules/typescript/lib/lib.webworker.importscripts.d.ts","../../../../node_modules/typescript/lib/lib.scripthost.d.ts","./math.ts","./text.ts"],"fileInfos":["2dc8c927c9c162a773c6bb3cdc4f3286c23f10eedc67414028f9cb5951610f60",{"version":"f20c05dbfe50a208301d2a1da37b9931bce0466eb5a1f4fe240971b4ecc82b67","affectsGlobalScope":true},{"version":"9b087de7268e4efc5f215347a62656663933d63c0b1d7b624913240367b999ea","affectsGlobalScope":true},{"version":"7fac8cb5fc820bc2a59ae11ef1c5b38d3832c6d0dfaec5acdb5569137d09a481","affectsGlobalScope":true},{"version":"097a57355ded99c68e6df1b738990448e0bf170e606707df5a7c0481ff2427cd","affectsGlobalScope":true},{"version":"6198e7d4a43aabb174a72ec9f0e8d2962912ad59ad90010aac3930868a8f62a4","signature":"0400cb85cef49e897c47df13e38b5cd199e0c900253f2d2ddf2e3491c27bc0a8"},{"version":"becd081df112726ab94c1ca1c05d6a59268fe0dabf7ad076d16ea851bf99e8fb","signature":"6039d94241358544e8d62a3a0ba90752a9973b3b2b422c187e2bcf7256fcda2e"}],"options":{"composite":true,"declaration":true,"declarationMap":true,"module":1,"noUnusedLocals":true,"noUnusedParameters":true,"strict":true,"target":1},"referencedMap":[],"exportedModulesMap":[],"semanticDiagnosticsPerFile":[6,7,1,3,2,5,4],"latestChangedDtsFile":"./text.d.ts"},"version":"4.8.4"}

e2e/test/typescript-project-references/stryker.conf.json

@@ -0,0 +1,17 @@
+{
+  "$schema": "../../node_modules/@stryker-mutator/core/schema/stryker-schema.json",
+  "packageManager": "npm",
+  "disableTypeChecks": true,
+  "testRunner": "mocha",
+  "concurrency": 1,
+  "coverageAnalysis": "perTest",
+  "reporters": ["json", "html", "progress", "clear-text"],
+  "checkers": ["typescript"],
+  "tsconfigFile": "src/core/tsconfig.json",
+  "fileLogLevel": "warn",
+  "buildCommand": "npm run build",
+  "plugins": [
+    "@stryker-mutator/mocha-runner",
+    "@stryker-mutator/typescript-checker"
+  ]
+}

e2e/test/typescript-project-references/test/job.spec.ts

@@ -0,0 +1,12 @@
+import { expect } from 'chai';
+import {start} from '../src/core/job';
+
+describe(start.name, () => {
+  it('should format a correct message', () => {
+    // Act
+    const result = start();
+
+    // Assert
+    expect(result).eq("Starting job");
+  });
+});

e2e/test/typescript-project-references/tsconfig.json

@@ -0,0 +1,3 @@
+{
+  "include": ["src/core", "src/utils", "test"]
+}

e2e/test/typescript-project-references/tsconfig.settings.json

@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "strict": true,
+    "target": "es5",
+    "moduleResolution": "node",
+    "module": "commonjs",
+    "composite": true,
+    "declaration": true,
+    "declarationMap": true,
+
+    // These settings should be overridden by the typescript checker
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+
+    "types": []
+  }
+}

e2e/test/typescript-project-references/verify/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "module"
+}

e2e/test/typescript-project-references/verify/verify.js

@@ -0,0 +1,7 @@
+import { expectMetricsJsonToMatchSnapshot } from '../../../helpers.js';
+
+describe('Verify stryker has ran correctly', () => {
+  it('should report correct score', async () => {
+    await expectMetricsJsonToMatchSnapshot();
+  });
+});

e2e/test/typescript-project-references/verify/verify.js.snap

@@ -0,0 +1,21 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`Verify stryker has ran correctly should report correct score 1`] = `
+Object {
+  "compileErrors": 3,
+  "ignored": 0,
+  "killed": 1,
+  "mutationScore": 33.33333333333333,
+  "mutationScoreBasedOnCoveredCode": 33.33333333333333,
+  "noCoverage": 0,
+  "runtimeErrors": 0,
+  "survived": 2,
+  "timeout": 0,
+  "totalCovered": 3,
+  "totalDetected": 1,
+  "totalInvalid": 3,
+  "totalMutants": 6,
+  "totalUndetected": 2,
+  "totalValid": 3,
+}
+`;

packages/typescript-checker/.vscode/launch.json

@@ -4,7 +4,24 @@
     {
       "type": "node",
       "request": "launch",
-      "name": "💙 Unit / Integration tests",
+      "name": "💙 Unit tests",
+      "program": "${workspaceRoot}/../../node_modules/mocha/bin/_mocha",
+      "internalConsoleOptions": "openOnSessionStart",
+      "outFiles": [
+        "${workspaceRoot}/dist/**/*.js"
+      ],
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "args": [
+        "--no-timeout",
+        "dist/test/unit/**/*.js",   
+      ]
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "💙 Integration tests",
       "program": "${workspaceRoot}/../../node_modules/mocha/bin/_mocha",
       "internalConsoleOptions": "openOnSessionStart",
       "outFiles": [
@@ -15,7 +32,6 @@
       ],
       "args": [
         "--no-timeout",
-        "dist/test/unit/**/*.js",
         "dist/test/integration/**/*.js"    
       ]
     }

packages/typescript-checker/schema/typescript-checker-options.json

@@ -0,0 +1,22 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema",
+  "title": "TypescriptCheckerPluginOptions",
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "typescriptChecker": {
+      "description": "Configuration for @stryker-mutator/typescript-checker",
+      "title": "TypescriptCheckerOptions",
+      "additionalProperties": false,
+      "type": "object",
+      "default": {},
+      "properties": {
+        "prioritizePerformanceOverAccuracy": {
+          "description": "Configures the performance of the TypescriptChecker. Setting this to false results in a slower, but more accurate result.",
+          "type": "boolean",
+          "default": true
+        }
+      }
+    }
+  }
+}

packages/typescript-checker/src/fs/hybrid-file-system.ts

@@ -1,5 +1,4 @@
 import ts from 'typescript';
-import { Mutant } from '@stryker-mutator/api/core';
 import { Logger } from '@stryker-mutator/api/logging';
 import { tokens, commonTokens } from '@stryker-mutator/api/plugin';
 
@@ -16,7 +15,6 @@ import { ScriptFile } from './script-file.js';
  */
 export class HybridFileSystem {
   private readonly files = new Map<string, ScriptFile | undefined>();
-  private mutatedFile: ScriptFile | undefined;
 
   public static inject = tokens(commonTokens.logger);
   constructor(private readonly log: Logger) {}
@@ -32,19 +30,6 @@ export class HybridFileSystem {
     }
   }
 
-  public mutate(mutant: Pick<Mutant, 'fileName' | 'location' | 'replacement'>): void {
-    const fileName = toPosixFileName(mutant.fileName);
-    const file = this.files.get(fileName);
-    if (!file) {
-      throw new Error(`File "${mutant.fileName}" cannot be found.`);
-    }
-    if (this.mutatedFile && this.mutatedFile !== file) {
-      this.mutatedFile.resetMutant();
-    }
-    file.mutate(mutant);
-    this.mutatedFile = file;
-  }
-
   public watchFile(fileName: string, watcher: ts.FileWatcherCallback): void {
     const file = this.getFile(fileName);
     if (file) {

packages/typescript-checker/src/fs/script-file.ts

@@ -45,7 +45,7 @@ export class ScriptFile {
     }
   }
 
-  private touch() {
+  private touch(): void {
     this.modifiedTime = new Date();
     this.watcher?.(this.fileName, ts.FileWatcherEventKind.Changed);
   }

packages/typescript-checker/src/grouping/create-groups.ts

@@ -0,0 +1,89 @@
+import { Mutant } from '@stryker-mutator/api/src/core/index.js';
+
+import { toPosixFileName } from '../tsconfig-helpers.js';
+
+import { TSFileNode } from './ts-file-node.js';
+
+/**
+ * To speed up the type-checking we want to check multiple mutants at once.
+ * When multiple mutants in different files don't have overlap in affected files (or have small overlap), we can type-check them simultaneously.
+ * These mutants who can be tested at the same time are called a group.
+ * Therefore, the return type is an array of arrays, in other words: an array of groups.
+ *
+ * @param mutants All the mutants of the test project.
+ * @param nodes A graph representation of the test project.
+ *
+ * @example
+ * Let's assume we got the following dependencies in files of a project, and in every file is one mutant.
+ *
+ *          ========
+ *          = A.ts =
+ *          ========
+ *         /        \
+ * ========          ========
+ * = B.ts =          = C.ts =
+ * ========          ========
+ *                           \
+ *                            ========
+ *                            = D.ts =
+ *                            ========
+ *
+ * A imports B and C
+ * C imports D
+ *
+ * In this example, we can type-check B and D simultaneously.
+ * This is because these files can't throw errors in each other.
+ * If we type check them, let's say B reports an error.
+ * We know that the mutant in B created the type error.
+ * If we type check B and D at the same time, it is possible that an error shows up in A.
+ * When this happens, we go down the dependency graph and individually test the mutants in that group.
+ *
+ * In this function, we create groups of mutants who can be tested at the same time.
+ */
+export function createGroups(mutants: Mutant[], nodes: Map<string, TSFileNode>): string[][] {
+  const groups: string[][] = [];
+  const mutantsToGroup = new Set(mutants);
+
+  while (mutantsToGroup.size) {
+    const group: string[] = [];
+    const groupNodes = new Set<TSFileNode>();
+    const nodesToIgnore = new Set<TSFileNode>();
+
+    for (const currentMutant of mutantsToGroup) {
+      const currentNode = findNode(currentMutant.fileName, nodes);
+      if (!nodesToIgnore.has(currentNode) && !parentsHaveOverlapWith(currentNode, groupNodes)) {
+        group.push(currentMutant.id);
+        groupNodes.add(currentNode);
+        mutantsToGroup.delete(currentMutant);
+        addRangeOfNodesToSet(nodesToIgnore, currentNode.getAllParentReferencesIncludingSelf());
+      }
+    }
+    groups.push(group);
+  }
+
+  return groups;
+}
+
+function addRangeOfNodesToSet(nodes: Set<TSFileNode>, nodesToAdd: Iterable<TSFileNode>) {
+  for (const parent of nodesToAdd) {
+    nodes.add(parent);
+  }
+}
+
+function findNode(fileName: string, nodes: Map<string, TSFileNode>) {
+  const node = nodes.get(toPosixFileName(fileName));
+  if (node == null) {
+    throw new Error(`Node not in graph: ${fileName}`);
+  }
+  return node;
+}
+
+function parentsHaveOverlapWith(currentNode: TSFileNode, groupNodes: Set<TSFileNode>) {
+  for (const parentNode of currentNode.getAllParentReferencesIncludingSelf()) {
+    if (groupNodes.has(parentNode)) {
+      return true;
+    }
+  }
+
+  return false;
+}