feat(core): add support for checker plugins (#2285)

Add support for Checker plugins. A checker plugin takes the place of the old transpiler plugin. When a checker plugin is configured, every mutant is checked before a test runner picks it up. This will allow typescript type checking to take place using the `@stryker-mutator/typescript-checker` plugin.

Since checking a mutant can take a long time (~2 sec/mutant with the `@stryker-mutator/typescript-checker` on Stryker's core project), we use worker processes to run this in parallel. These processes are entirely managed by Stryker core, plugin creators don't need to worry about this.

Add `--concurrency` (or `-c`) option which set the number of concurrent worker processes to be used by Stryker. A worker can be a checker process or a test runner process. The default is nr of CPU's available -1, with a minimum of 1 test runner process and 1 checker process (if a checker is configured).

The algorithm used to determine the number of checkers is `ceil(--concurrency / 2)`. This allows room for `floor(--concurrency / 2)` test runners. Once the check process is done, all check processes are closed and more test runner processes are spun up to speed up the mutation testing process.

BREAKING CHANGE:

* `--maxConcurrentTestRunners` is now deprecated. Please use `--concurrency` instead.

e2e/package.json

@@ -16,6 +16,7 @@
     "chai-as-promised": "~7.1.1",
     "cross-env": "~5.2.0",
     "grunt": "~1.0.4",
+    "jasmine": "^3.5.0",
     "jasmine-core": "~3.5.0",
     "jest": "^26.0.1",
     "karma": "^5.0.4",
@@ -57,6 +58,7 @@
     "@stryker-mutator/mocha-runner": "../packages/mocha-runner",
     "@stryker-mutator/util": "../packages/util",
     "@stryker-mutator/typescript": "../packages/typescript",
+    "@stryker-mutator/typescript-checker": "../packages/typescript-checker",
     "@stryker-mutator/vue-mutator": "../packages/vue-mutator",
     "@stryker-mutator/webpack-transpiler": "../packages/webpack-transpiler"
   }

e2e/tasks/run-e2e-tests.ts

@@ -9,6 +9,7 @@ import { tap, mergeAll, map, filter } from 'rxjs/operators';
 const testRootDir = path.resolve(__dirname, '..', 'test');
 
 const mutationSwitchingTempWhiteList = [
+  'jasmine-ts-node',
   'jasmine-jasmine',
   'karma-mocha',
   'karma-jasmine',

e2e/test/jasmine-ts-node/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "jasmine-ts-node",
+  "description": "A test for mocha using ts-node (no transpiler) using mocha package config",
+  "scripts": {
+    "test:unit": "jasmine",
+    "test": "stryker run",
+    "posttest": "mocha --no-package --require \"ts-node/register\" verify/verify.ts"
+  },
+  "mocha": {
+    "require": "ts-node/register/transpile-only",
+    "spec": "test/**/*.ts"
+  },
+  "devDependencies": {
+    "@types/jasmine": "^3.5.11"
+  }
+}

e2e/test/jasmine-ts-node/spec/PlayerSpec.ts

@@ -0,0 +1,61 @@
+import { Player } from '../src/Player';
+import { Song } from '../src/Song';
+
+describe("Player", function () {
+  var player;
+  var song;
+
+  beforeEach(function () {
+    player = new Player();
+    song = new Song();
+  });
+
+  it("should be able to play a Song", function () {
+    player.play(song);
+    expect(player.currentlyPlayingSong).toEqual(song);
+
+    //demonstrates use of custom matcher
+    expect(player).toBePlaying(song);
+  });
+
+  describe("when song has been paused", function () {
+    beforeEach(function () {
+      player.play(song);
+      player.pause();
+    });
+
+    it("should indicate that the song is currently paused", function () {
+      expect(player.isPlaying).toBeFalsy();
+
+      // demonstrates use of 'not' with a custom matcher
+      expect(player).not.toBePlaying(song);
+    });
+
+    it("should be possible to resume", function () {
+      player.resume();
+      expect(player.isPlaying).toBeTruthy();
+      expect(player.currentlyPlayingSong).toEqual(song);
+    });
+  });
+
+  // demonstrates use of spies to intercept and test method calls
+  it("tells the current song if the user has made it a favorite", function () {
+    spyOn(song, 'persistFavoriteStatus');
+
+    player.play(song);
+    player.makeFavorite();
+
+    expect(song.persistFavoriteStatus).toHaveBeenCalledWith(true);
+  });
+
+  //demonstrates use of expected exceptions
+  describe("#resume", function () {
+    it("should throw an exception if song is already playing", function () {
+      player.play(song);
+
+      expect(function () {
+        player.resume();
+      }).toThrowError("song is already playing");
+    });
+  });
+});

e2e/test/jasmine-ts-node/spec/concat.spec.ts

@@ -0,0 +1,7 @@
+import { concat } from '../src/concat';
+
+describe(concat.name, () => {
+  it('should concat a and b', () => {
+    expect(concat('foo', 'bar')).toBe('foobar');
+  })
+});

e2e/test/jasmine-ts-node/spec/helpers/1-register-ts-node.js

@@ -0,0 +1,3 @@
+require('ts-node').register({  
+  transpileOnly: true
+});

e2e/test/jasmine-ts-node/spec/helpers/CustomMatchers.ts

@@ -0,0 +1,5 @@
+declare module jasmine {
+  interface Matchers<T> {
+    toBePlaying(song: import('../../src/Song').Song): boolean;
+  }
+}

e2e/test/jasmine-ts-node/spec/helpers/IsCurrentlyPlayingMatcher.ts

@@ -0,0 +1,20 @@
+import { Player } from "../../src/Player";
+import { Song } from "../../src/Song";
+
+beforeEach(function () {
+  jasmine.addMatchers({
+    toBePlaying: function () {
+      return {
+        compare: function (actual: Player, expected: Song) {
+          var player = actual;
+
+          return {
+            pass: player.currentlyPlayingSong === expected && player.isPlaying
+          }
+        }
+      };
+    }
+  });
+});
+
+

e2e/test/jasmine-ts-node/spec/support/jasmine.json

@@ -0,0 +1,11 @@
+{
+  "spec_dir": "spec",
+  "spec_files": [
+    "**/*[sS]pec.[jt]s"
+  ],
+  "helpers": [
+    "helpers/**/*.[jt]s"
+  ],
+  "stopSpecOnExpectationFailure": false,
+  "random": true
+}

e2e/test/jasmine-ts-node/src/Player.ts

@@ -0,0 +1,26 @@
+import { Song } from "./Song";
+
+export class Player {
+  public currentlyPlayingSong: Song;
+  public isPlaying: boolean;
+
+  play(song: Song) {
+    this.currentlyPlayingSong = song;
+    this.isPlaying = true;
+  }
+
+  pause() {
+    this.isPlaying = false;
+  }
+
+  resume() {
+    if (this.isPlaying) {
+      throw new Error('song is already playing');
+    }
+    this.isPlaying = true;
+  }
+
+  makeFavorite() {
+    this.currentlyPlayingSong.persistFavoriteStatus(true);
+  }
+}

e2e/test/jasmine-ts-node/src/Song.ts

@@ -0,0 +1,5 @@
+export class Song {
+  persistFavoriteStatus(value: boolean) {
+    throw new Error('not yet implemented');
+  }
+}

e2e/test/jasmine-ts-node/src/concat.ts

@@ -0,0 +1,3 @@
+export function concat(a: string, b: string) {
+  return a + b; // Will be a compile error when mutated
+}

e2e/test/jasmine-ts-node/stryker.conf.json

@@ -0,0 +1,17 @@
+{
+  "$schema": "../../node_modules/@stryker-mutator/core/schema/stryker-schema.json",
+  "mutate": [
+    "{src,lib}/**/*.ts?(x)"
+  ],
+  "checkers": [
+    "typescript"
+  ],
+  "testRunner": "jasmine",
+  "concurrency": 2,
+  "coverageAnalysis": "perTest",
+  "reporters": ["event-recorder", "clear-text"],
+  "plugins": [
+    "@stryker-mutator/typescript-checker",
+    "@stryker-mutator/jasmine-runner"
+  ]
+}

e2e/test/jasmine-ts-node/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "compilerOptions": {
+    "lib": ["ES2017"],
+    "module": "CommonJS",
+    "target": "ES2017",
+    "types": ["jasmine"]
+  },
+  "include": ["src", "spec"]
+}

e2e/test/jasmine-ts-node/verify/verify.ts

@@ -0,0 +1,17 @@
+import { expectMetrics } from '../../../helpers';
+
+describe('Verify stryker has ran correctly', () => {
+
+  it('should report correct score', async () => {
+    await expectMetrics({
+      ignored: 0,
+      killed: 12,
+      mutationScore: 85.71,
+      noCoverage: 2,
+      survived: 0,
+      timeout: 0,
+      runtimeErrors: 0,
+      compileErrors: 2
+    });
+  });
+});

packages/api/schema/stryker-core.json

@@ -209,6 +209,18 @@
       "type": "boolean",
       "default": true
     },
+    "checkers": {
+      "description": "Enable checker plugins here. A checker plugin will be invoked for each mutant before it is run in a test runner. It can check to see of a given mutant is valid, by for example validate that it won't result in a type error",
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "default": []
+    },
+    "concurrency": {
+      "description": "Set the concurrency of workers. Stryker will always run checkers and test runners in parallel by creating worker processes (note, not `worker_threads`). This defaults to n-1 where n is the number of cpu's available on your machine. This is a sane default for most use cases.",
+      "type": "number"
+    },
     "coverageAnalysis": {
       "$ref": "#/definitions/coverageAnalysis",
       "description": "Indicates which coverage analysis strategy to use. During mutation testing, stryker will try to only run the tests that cover a particular line of code.\n\n'perTest': Analyse coverage per test.\n'all': Analyse the coverage for the entire test suite.\n'off' (default): Don't use coverage analysis",
@@ -247,7 +259,7 @@
       "default": "info"
     },
     "maxConcurrentTestRunners": {
-      "description": "Specifies the maximum number of concurrent test runners to spawn. Mutation testing is time consuming. By default, Stryker tries to make the most of your CPU's, by spawning as many test runners as you have CPU cores (`Number.MAX_SAFE_INTEGER`).",
+      "description": "[DEPRECATED please use \"concurrency\" instead]. Specifies the maximum number of concurrent test runners to spawn. Mutation testing is time consuming. By default, Stryker tries to make the most of your CPU's, by spawning as many test runners as you have CPU cores (`Number.MAX_SAFE_INTEGER`).",
       "type": "number",
       "default": 9007199254740991
     },

packages/api/src/test_runner2/TestRunner.ts

@@ -3,8 +3,8 @@ import { DryRunResult } from './DryRunResult';
 import { MutantRunResult } from './MutantRunResult';
 
 export interface TestRunner2 {
-  init?(): Promise<void> | void;
+  init?(): Promise<void>;
   dryRun(options: DryRunOptions): Promise<DryRunResult>;
   mutantRun(options: MutantRunOptions): Promise<MutantRunResult>;
-  dispose?(): Promise<void> | void;
+  dispose?(): Promise<void>;
 }

packages/core/find.js

@@ -0,0 +1,26 @@
+const { zip, range, Subject, partition, ReplaySubject } = require('rxjs');
+const { tap, shareReplay } = require('rxjs/operators');
+
+const subject = new ReplaySubject();
+
+subject.next(0);
+
+const n$ = subject.pipe(tap(n => console.log(`before replay ${n}`)), shareReplay(), tap(n => console.log(`after replay ${n}`)))
+
+
+const sub1 = n$.subscribe(console.log.bind(console, 'sub1'));
+
+subject.next(1);
+
+const sub2 = n$.subscribe(console.log.bind(console, 'sub2'))
+
+
+subject.next(2);
+
+sub1.unsubscribe();
+sub2.unsubscribe();
+
+subject.next(3);
+n$.subscribe(console.log.bind(console, 'sub3'));
+
+subject.next(4);

packages/core/package.json

@@ -63,7 +63,6 @@
     "get-port": "~5.0.0",
     "glob": "~7.1.2",
     "inquirer": "~7.2.0",
-    "istanbul-lib-instrument": "~3.3.0",
     "lodash.flatmap": "^4.5.0",
     "lodash.groupby": "^4.6.0",
     "log4js": "6.2.1",
@@ -84,7 +83,6 @@
     "@stryker-mutator/api": "^0.24.1",
     "@stryker-mutator/test-helpers": "^3.3.0",
     "@types/inquirer": "~6.0.2",
-    "@types/istanbul-lib-instrument": "~1.7.0",
     "@types/lodash.flatmap": "^4.5.6",
     "@types/lodash.groupby": "^4.6.6",
     "@types/node": "^14.0.1",