feat(instrumenter): add excludedMutations support (#2513)

Add support to configure `mutator.excludedMutations`. This was already supported in Stryker@<3, but not yet in Stryker4 beta.

Instead of not generating the mutants, it generates them without placing them in the code. They get reported as `ignored` in your mutation testing report.

![image](https://user-images.githubusercontent.com/1828233/94722119-2142dc00-0357-11eb-839c-a4aef430dcea.png)

Also, add error message, ignore reason, and "killed by" test name in the "show more" modal dialog.

![image](https://user-images.githubusercontent.com/1828233/94722368-8565a000-0357-11eb-9a2d-05be1ac0d161.png)

e2e/test/ignore-project/.mocharc.jsonc

@@ -0,0 +1,4 @@
+{
+    "require": "./test/helpers/testSetup.js",
+    "spec": ["test/unit/*.js"]
+}

e2e/test/ignore-project/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "ignore-project",
+  "description": "e2e test for different ignored mutants",
+  "version": "0.0.0",
+  "private": true,
+  "scripts": {
+    "test:unit": "mocha",
+    "test": "stryker run",
+    "posttest": "mocha --no-config --require ../../tasks/ts-node-register.js verify/*.ts"
+  }
+}

e2e/test/ignore-project/src/Add.js

@@ -0,0 +1,26 @@
+module.exports.add = function(num1, num2) {
+  return num1 + num2;
+};
+
+module.exports.addOne = function(number) {
+  number++;
+  return number;
+};
+
+module.exports.negate = function(number) {
+  return -number;
+};
+
+module.exports.notCovered = function(number) {
+  return number > 10;
+};
+
+module.exports.isNegativeNumber = function(number) {
+  var isNegative = false;
+  if(number < 0){
+    isNegative = true;
+  }
+  return isNegative;
+};
+
+

e2e/test/ignore-project/src/Circle.js

@@ -0,0 +1,8 @@
+module.exports.getCircumference = function(radius) {
+  //Function to test multiple math mutations in a single function.
+  return 2 * Math.PI * radius;
+};
+
+module.exports.untestedFunction = function() {
+  var i = 5 / 2 * 3;
+};

e2e/test/ignore-project/stryker.conf.json

@@ -0,0 +1,18 @@
+{
+  "$schema": "../../node_modules/@stryker-mutator/core/schema/stryker-schema.json",
+  "testRunner": "mocha",
+  "concurrency": 2,
+  "coverageAnalysis": "perTest",
+  "mutator": {
+    "excludedMutations": ["ArithmeticOperator", "BlockStatement"]
+  },
+  "reporters": [
+    "clear-text",
+    "html",
+    "event-recorder"
+  ],
+  "plugins": [
+    "@stryker-mutator/mocha-runner"
+  ],
+  "allowConsoleColors": false
+}

e2e/test/ignore-project/test/helpers/testSetup.js

@@ -0,0 +1,5 @@
+exports.mochaHooks = {
+  beforeAll() {
+    global.expect = require('chai').expect;
+  }
+}

e2e/test/ignore-project/test/unit/AddSpec.js

@@ -0,0 +1,48 @@
+const { expect } = require('chai');
+const { add, addOne, isNegativeNumber, negate, notCovered } = require('../../src/Add');
+
+describe('Add', function () {
+  it('should be able to add two numbers', function () {
+    var num1 = 2;
+    var num2 = 5;
+    var expected = num1 + num2;
+
+    var actual = add(num1, num2);
+
+    expect(actual).to.be.equal(expected);
+  });
+
+  it('should be able 1 to a number', function () {
+    var number = 2;
+    var expected = 3;
+
+    var actual = addOne(number);
+
+    expect(actual).to.be.equal(expected);
+  });
+
+  it('should be able negate a number', function () {
+    var number = 2;
+    var expected = -2;
+
+    var actual = negate(number);
+
+    expect(actual).to.be.equal(expected);
+  });
+
+  it('should be able to recognize a negative number', function () {
+    var number = -2;
+
+    var isNegative = isNegativeNumber(number);
+
+    expect(isNegative).to.be.true;
+  });
+
+  it('should be able to recognize that 0 is not a negative number', function () {
+    var number = 0;
+
+    var isNegative = isNegativeNumber(number);
+
+    expect(isNegative).to.be.false;
+  });
+});

e2e/test/ignore-project/test/unit/CircleSpec.js

@@ -0,0 +1,13 @@
+const { expect } = require('chai');
+const { getCircumference } = require('../../src/Circle');
+
+describe('Circle', function () {
+  it('should have a circumference of 2PI when the radius is 1', function () {
+    var radius = 1;
+    var expectedCircumference = 2 * Math.PI;
+
+    var circumference = getCircumference(radius);
+
+    expect(circumference).to.be.equal(expectedCircumference);
+  });
+});

e2e/test/ignore-project/verify/verify.ts

@@ -0,0 +1,16 @@
+import { expectMetrics } from '../../../helpers';
+
+describe('After running stryker on jest-react project', () => {
+  it('should report expected scores', async () => {
+    await expectMetrics({
+      killed: 8,
+      ignored: 13,
+      mutationScore: 66.67,
+    });
+    /*
+-----------|---------|----------|-----------|------------|----------|---------|
+File       | % score | # killed | # timeout | # survived | # no cov | # error |
+-----------|---------|----------|-----------|------------|----------|---------|
+All files  |   66.67 |        8 |         0 |          0 |        4 |       0 |*/
+  });
+});

packages/api/src/core/Mutant.ts

@@ -1,13 +1,38 @@
 import Range from './Range';
 import Location from './Location';
 
+/**
+ * Represents a mutant
+ */
 interface Mutant {
+  /**
+   * The id of the mutant. Unique within a run.
+   */
   id: number;
+  /**
+   * The name of the mutator that generated this mutant.
+   */
   mutatorName: string;
+  /**
+   * The file name from which this mutant originated
+   */
   fileName: string;
+  /**
+   * The range of this mutant (from/to within the file)
+   */
   range: Range;
+  /**
+   * The line number/column location of this mutant
+   */
   location: Location;
+  /**
+   * The replacement (actual mutated code)
+   */
   replacement: string;
+  /**
+   * If the mutant was ignored during generation, the reason for ignoring it, otherwise `undefined`
+   */
+  ignoreReason?: string;
 }
 
 export default Mutant;

packages/api/src/core/Range.ts

@@ -1,6 +1,6 @@
 /**
- * Represents a location in a file by range [fromInclusive, toExclusive]
+ * Represents a location in a file by range.
  */
-type Range = [number, number];
+type Range = [fromInclusive: number, toExclusive: number];
 
 export default Range;

packages/api/src/report/MutantResult.ts

@@ -24,6 +24,11 @@ export interface InvalidMutantResult extends BaseMutantResult {
   errorMessage: string;
 }
 
+export interface IgnoredMutantResult extends BaseMutantResult {
+  status: MutantStatus.Ignored;
+  ignoreReason: string;
+}
+
 export interface UndetectedMutantResult extends BaseMutantResult {
   status: MutantStatus.NoCoverage | MutantStatus.Survived;
   testFilter: string[] | undefined;
@@ -33,4 +38,4 @@ export interface TimeoutMutantResult extends BaseMutantResult {
   status: MutantStatus.TimedOut;
 }
 
-export type MutantResult = TimeoutMutantResult | UndetectedMutantResult | InvalidMutantResult | KilledMutantResult;
+export type MutantResult = TimeoutMutantResult | UndetectedMutantResult | InvalidMutantResult | KilledMutantResult | IgnoredMutantResult;

packages/api/src/report/MutantStatus.ts

@@ -36,6 +36,11 @@ enum MutantStatus {
    * ```
    */
   CompileError,
+
+  /**
+   * The status of a mutant that is ignored. For example, by user configuration.
+   */
+  Ignored,
 }
 
 export default MutantStatus;

packages/core/src/mutants/findMutantTestCoverage.ts

@@ -45,7 +45,13 @@ function mapToMutantTestCoverage(dryRunResult: CompleteDryRunResult, mutants: re
   const timeSpentAllTests = calculateTotalTime(dryRunResult.tests);
 
   const mutantCoverage = mutants.map((mutant) => {
-    if (!dryRunResult.mutantCoverage || dryRunResult.mutantCoverage.static[mutant.id] > 0) {
+    if (mutant.ignoreReason !== undefined) {
+      return {
+        mutant,
+        estimatedNetTime: 0,
+        coveredByTests: false,
+      };
+    } else if (!dryRunResult.mutantCoverage || dryRunResult.mutantCoverage.static[mutant.id] > 0) {
       return {
         mutant,
         estimatedNetTime: timeSpentAllTests,

packages/core/src/process/2-MutantInstrumenterExecutor.ts

@@ -31,7 +31,7 @@ export class MutantInstrumenterExecutor {
     const instrumenter = this.injector.injectClass(Instrumenter);
 
     // Instrument files in-memory
-    const instrumentResult = await instrumenter.instrument(this.inputFiles.filesToMutate, { plugins: this.options.mutator.plugins });
+    const instrumentResult = await instrumenter.instrument(this.inputFiles.filesToMutate, this.options.mutator);
 
     // Preprocess sandbox files
     const preprocess = this.injector.injectFunction(createPreprocessor);

packages/core/src/process/4-MutationTestExecutor.ts

@@ -6,17 +6,14 @@ import { MutantResult } from '@stryker-mutator/api/report';
 import { MutantRunOptions, TestRunner } from '@stryker-mutator/api/test_runner';
 import { Logger } from '@stryker-mutator/api/logging';
 import { I } from '@stryker-mutator/util';
-
 import { CheckStatus, Checker, CheckResult, PassedCheckResult } from '@stryker-mutator/api/check';
 
 import { coreTokens } from '../di';
 import StrictReporter from '../reporters/StrictReporter';
 import { MutantTestCoverage } from '../mutants/findMutantTestCoverage';
 import { MutationTestReportHelper } from '../reporters/MutationTestReportHelper';
 import Timer from '../utils/Timer';
-
 import { Pool, ConcurrencyTokenProvider } from '../concurrent';
-
 import { Sandbox } from '../sandbox';
 
 import { DryRunContext } from './3-DryRunExecutor';
@@ -58,22 +55,27 @@ export class MutationTestExecutor {
   ) {}
 
   public async execute(): Promise<MutantResult[]> {
-    const { passedMutant$, checkResult$ } = this.executeCheck(from(this.matchedMutants));
+    const { ignoredResult$, notIgnoredMutant$ } = this.executeIgnore(from(this.matchedMutants));
+    const { passedMutant$, checkResult$ } = this.executeCheck(from(notIgnoredMutant$));
     const { coveredMutant$, noCoverageResult$ } = this.executeNoCoverage(passedMutant$);
     const testRunnerResult$ = this.executeRunInTestRunner(coveredMutant$);
-    const results = await merge(testRunnerResult$, checkResult$, noCoverageResult$).pipe(toArray()).toPromise();
+    const results = await merge(testRunnerResult$, checkResult$, noCoverageResult$, ignoredResult$).pipe(toArray()).toPromise();
     this.mutationTestReportHelper.reportAll(results);
     await this.reporter.wrapUp();
     this.logDone();
     return results;
   }
 
+  private executeIgnore(input$: Observable<MutantTestCoverage>) {
+    const [notIgnoredMutant$, ignoredMutant$] = partition(input$.pipe(shareReplay()), ({ mutant }) => mutant.ignoreReason === undefined);
+    const ignoredResult$ = ignoredMutant$.pipe(map(({ mutant }) => this.mutationTestReportHelper.reportMutantIgnored(mutant)));
+    return { ignoredResult$, notIgnoredMutant$ };
+  }
+
   private executeNoCoverage(input$: Observable<MutantTestCoverage>) {
     const [coveredMutant$, noCoverageMatchedMutant$] = partition(input$.pipe(shareReplay()), (matchedMutant) => matchedMutant.coveredByTests);
-    return {
-      noCoverageResult$: noCoverageMatchedMutant$.pipe(map(({ mutant }) => this.mutationTestReportHelper.reportNoCoverage(mutant))),
-      coveredMutant$,
-    };
+    const noCoverageResult$ = noCoverageMatchedMutant$.pipe(map(({ mutant }) => this.mutationTestReportHelper.reportNoCoverage(mutant)));
+    return { noCoverageResult$, coveredMutant$ };
   }
 
   private executeCheck(input$: Observable<MutantTestCoverage>) {
@@ -95,33 +97,20 @@ export class MutationTestExecutor {
         },
       })
     );
-    const [passedMutant$, failedMutant$] = partition(checkTask$.pipe(shareReplay()), ({ checkResult }) => {
-      return checkResult.status === CheckStatus.Passed;
-    });
-    return {
-      checkResult$: failedMutant$.pipe(
-        map((failedMutant) =>
-          this.mutationTestReportHelper.reportCheckFailed(
-            failedMutant.matchedMutant.mutant,
-            failedMutant.checkResult as Exclude<CheckResult, PassedCheckResult>
-          )
+    const [passedCheckResult$, failedCheckResult$] = partition(
+      checkTask$.pipe(shareReplay()),
+      ({ checkResult }) => checkResult.status === CheckStatus.Passed
+    );
+    const checkResult$ = failedCheckResult$.pipe(
+      map((failedMutant) =>
+        this.mutationTestReportHelper.reportCheckFailed(
+          failedMutant.matchedMutant.mutant,
+          failedMutant.checkResult as Exclude<CheckResult, PassedCheckResult>
         )
-      ),
-      passedMutant$: passedMutant$.pipe(map(({ matchedMutant }) => matchedMutant)),
-    };
-  }
-
-  private createMutantRunOptions(mutant: MutantTestCoverage): MutantRunOptions {
-    return {
-      activeMutant: mutant.mutant,
-      timeout: this.calculateTimeout(mutant),
-      testFilter: mutant.testFilter,
-      sandboxFileName: this.sandbox.sandboxFileFor(mutant.mutant.fileName),
-    };
-  }
-
-  private calculateTimeout(mutant: MutantTestCoverage) {
-    return this.options.timeoutFactor * mutant.estimatedNetTime + this.options.timeoutMS + this.timeOverheadMS;
+      )
+    );
+    const passedMutant$ = passedCheckResult$.pipe(map(({ matchedMutant }) => matchedMutant));
+    return { checkResult$, passedMutant$ };
   }
 
   private executeRunInTestRunner(input$: Observable<MutantTestCoverage>): Observable<MutantResult> {
@@ -135,6 +124,16 @@ export class MutationTestExecutor {
     );
   }
 
+  private createMutantRunOptions(mutant: MutantTestCoverage): MutantRunOptions {
+    const timeout = this.options.timeoutFactor * mutant.estimatedNetTime + this.options.timeoutMS + this.timeOverheadMS;
+    return {
+      activeMutant: mutant.mutant,
+      timeout: timeout,
+      testFilter: mutant.testFilter,
+      sandboxFileName: this.sandbox.sandboxFileFor(mutant.mutant.fileName),
+    };
+  }
+
   private logDone() {
     this.log.info('Done in %s.', this.timer.humanReadableElapsed());
   }