feat(Ignorer plugin): support ignorer plugins (#4487)

Add a new plugin type: "Ignorer". Ignorer plugins can ignore mutants based on arbitrary heuristics.

You can configure it like this:

```json
{
  "ignorers": ["ConsoleIgnorer"],
  "plugins": [
    "@stryker-mutator/*",
    "./stryker-plugins/console-ignorer.js"
  ]
}
```

The console-ignorer.js file might look like this:

```js
import { PluginKind, declareClassPlugin } from '@stryker-mutator/api/plugin';

class ConsoleIgnorer {
  shouldIgnore(path) {
    if (
      path.isExpressionStatement() &&
      path.node.expression.type === 'CallExpression' &&
      path.node.expression.callee.type === 'MemberExpression' &&
      path.node.expression.callee.object.type === 'Identifier' &&
      path.node.expression.callee.object.name === 'console' &&
      path.node.expression.callee.property.type === 'Identifier' &&
      path.node.expression.callee.property.name === 'log'
    ) {
      return "We're not interested in console.log statements for now";
    }
    return undefined;
  }
}
export const strykerPlugins = [declareClassPlugin(PluginKind.Ignorer, 'ConsoleIgnorer', ConsoleIgnorer)];
```

---------

Co-authored-by: Nico Jansen <jansennico@gmail.com>

.prettierrc

@@ -1,5 +1,5 @@
 {
-  'printWidth': 150,
-  'singleQuote': true,
-  'endOfLine': 'auto',
+  "printWidth": 150,
+  "singleQuote": true,
+  "endOfLine": "auto"
 }

e2e/test/ignore-project/package.json

@@ -3,6 +3,7 @@
   "description": "e2e test for different ignored mutants",
   "version": "0.0.0",
   "private": true,
+  "type": "module",
   "scripts": {
     "test:unit": "mocha",
     "test": "stryker run",

e2e/test/ignore-project/src/Add.js

@@ -1,41 +1,45 @@
-module.exports.add = function (num1, num2) {
+export function add (num1, num2) {
+  console.log('Add' + 'called');
   return num1 + num2;
-};
+}
 
-module.exports.addOne = function (number) {
+export function addOne (number) {
   number++;
+  console.log('add one called');
   return number;
-};
+}
 
-module.exports.negate = function (number) {
+export function negate (number) {
+  console.log('negate called');
   return -number;
-};
+}
 
-module.exports.notCovered = function (number) {
+export function notCovered (number) {
+  console.warn('not covered!');
   return number > 10;
-};
+}
 
-module.exports.userNextLineIgnored = function (number) {
+export function userNextLineIgnored (number) {
   // Stryker disable next-line all: Ignoring this on purpose
   return number > 10;
-};
+}
 
 // Stryker disable all
-module.exports.blockUserIgnored = function (number) {
+export function blockUserIgnored (number) {
   return number > 10;
-};
+}
 // Stryker restore all
 
-module.exports.userNextLineSpecificMutator = function (number) {
+export function userNextLineSpecificMutator (number) {
   // Stryker disable next-line BooleanLiteral, ConditionalExpression: Ignore boolean and conditions
   return true && number > 10;
-};
+}
 
 
-module.exports.isNegativeNumber = function (number) {
+export function isNegativeNumber (number) {
   var isNegative = false;
   if (number < 0) {
     isNegative = true;
   }
   return isNegative;
-};
+}

e2e/test/ignore-project/src/Circle.js

@@ -1,8 +1,8 @@
-module.exports.getCircumference = function(radius) {
+export function getCircumference(radius) {
   //Function to test multiple math mutations in a single function.
   return 2 * Math.PI * radius;
-};
+}
 
-module.exports.untestedFunction = function() {
+export function untestedFunction() {
   var i = 5 / 2 * 3;
-};
+}

e2e/test/ignore-project/stryker-plugins/ignorers/console-ignorer.js

@@ -0,0 +1,23 @@
+// @ts-check
+import { PluginKind, declareClassPlugin } from '@stryker-mutator/api/plugin';
+
+export class ConsoleIgnorer {
+  /**
+   * @param {import('@stryker-mutator/api/ignorer').NodePath} path
+   */
+  shouldIgnore(path) {
+    if (
+      path.isExpressionStatement() &&
+      path.node.expression.type === 'CallExpression' &&
+      path.node.expression.callee.type === 'MemberExpression' &&
+      path.node.expression.callee.object.type === 'Identifier' &&
+      path.node.expression.callee.object.name === 'console' &&
+      path.node.expression.callee.property.type === 'Identifier' &&
+      path.node.expression.callee.property.name === 'log'
+    ) {
+      return "We're not interested in console.log statements for now";
+    }
+    return undefined;
+  }
+}
+export const strykerPlugins = [declareClassPlugin(PluginKind.Ignorer, 'ConsoleIgnorer', ConsoleIgnorer)];

e2e/test/ignore-project/stryker-plugins/ignorers/global.types.ts

@@ -0,0 +1,6 @@
+import type babel from '@babel/core';
+
+declare module '@stryker-mutator/api/ignorer' {
+  // eslint-disable-next-line @typescript-eslint/no-empty-interface
+  interface NodePath extends babel.NodePath {}
+}

e2e/test/ignore-project/stryker.conf.json

@@ -12,8 +12,10 @@
     "json",
     "event-recorder"
   ],
+  "ignorers": ["ConsoleIgnorer"],
   "plugins": [
-    "@stryker-mutator/mocha-runner"
+    "@stryker-mutator/mocha-runner",
+    "./stryker-plugins/ignorers/console-ignorer.js"
   ],
   "allowConsoleColors": false
 }

e2e/test/ignore-project/test/helpers/testSetup.js

@@ -1,5 +1,6 @@
-exports.mochaHooks = {
+import { expect } from 'chai';
+export const mochaHooks = {
   beforeAll() {
-    global.expect = require('chai').expect;
+    global.expect = expect;
   }
 }

e2e/test/ignore-project/test/unit/AddSpec.js

@@ -1,5 +1,5 @@
-const { expect } = require('chai');
-const { add, addOne, isNegativeNumber, negate, notCovered } = require('../../src/Add');
+import { expect } from 'chai';
+import { add, addOne, isNegativeNumber, negate } from '../../src/Add.js';
 
 describe('Add', function () {
   it('should be able to add two numbers', function () {

e2e/test/ignore-project/test/unit/CircleSpec.js

@@ -1,5 +1,5 @@
-const { expect } = require('chai');
-const { getCircumference } = require('../../src/Circle');
+import { expect } from 'chai';
+import { getCircumference } from '../../src/Circle.js';
 
 describe('Circle', function () {
   it('should have a circumference of 2PI when the radius is 1', function () {

e2e/test/ignore-project/verify/verify.js

@@ -1,19 +1,23 @@
 import { expect } from 'chai';
 import { MutantStatus } from 'mutation-testing-report-schema';
 
-import { expectMetricsJsonToMatchSnapshot, readMutationTestingJsonResultAsMetricsResult } from '../../../helpers.js';
+import { expectMetricsJsonToMatchSnapshot, readMutationTestingJsonResult } from '../../../helpers.js';
 
 describe('After running stryker on jest-react project', () => {
   it('should report expected scores', async () => {
     await expectMetricsJsonToMatchSnapshot();
   });
+
   it('should report mutants that are disabled by a comment with correct ignore reason', async () => {
-    const actualMetricsResult = await readMutationTestingJsonResultAsMetricsResult();
-    const addResult = actualMetricsResult.systemUnderTestMetrics.childResults.find((file) => file.name.endsWith('Add.js')).file;
-    const mutantsAtLine31 = addResult.mutants.filter(({ location }) => location.start.line === 31);
-    const booleanLiteralMutants = mutantsAtLine31.filter(({ mutatorName }) => mutatorName === 'BooleanLiteral');
-    const conditionalExpressionMutants = mutantsAtLine31.filter(({ mutatorName }) => mutatorName === 'ConditionalExpression');
-    const equalityOperatorMutants = mutantsAtLine31.filter(({ mutatorName }) => mutatorName === 'EqualityOperator');
+    const report = await readMutationTestingJsonResult();
+    const addResult = report.files['src/Add.js'];
+    const mutantsAtLine35 = addResult.mutants.filter(({ location }) => location.start.line === 35);
+    const booleanLiteralMutants = mutantsAtLine35.filter(({ mutatorName }) => mutatorName === 'BooleanLiteral');
+    const conditionalExpressionMutants = mutantsAtLine35.filter(({ mutatorName }) => mutatorName === 'ConditionalExpression');
+    const equalityOperatorMutants = mutantsAtLine35.filter(({ mutatorName }) => mutatorName === 'EqualityOperator');
+    expect(booleanLiteralMutants).lengthOf(1);
+    expect(conditionalExpressionMutants).lengthOf(3);
+    expect(equalityOperatorMutants).lengthOf(2);
     booleanLiteralMutants.forEach((booleanMutant) => {
       expect(booleanMutant.status).eq(MutantStatus.Ignored);
       expect(booleanMutant.statusReason).eq('Ignore boolean and conditions');
@@ -26,13 +30,34 @@ describe('After running stryker on jest-react project', () => {
       expect(equalityMutant.status).eq(MutantStatus.NoCoverage);
     });
   });
+
   it('should report mutants that result from excluded mutators with the correct ignore reason', async () => {
-    const actualMetricsResult = await readMutationTestingJsonResultAsMetricsResult();
-    const circleResult = actualMetricsResult.systemUnderTestMetrics.childResults.find((file) => file.name.endsWith('Circle.js')).file;
+    const report = await readMutationTestingJsonResult();
+    const circleResult = report.files['src/Circle.js'];
     const mutantsAtLine3 = circleResult.mutants.filter(({ location }) => location.start.line === 3);
+    expect(mutantsAtLine3).lengthOf(2);
     mutantsAtLine3.forEach((mutant) => {
       expect(mutant.status).eq(MutantStatus.Ignored);
       expect(mutant.statusReason).eq('Ignored because of excluded mutation "ArithmeticOperator"');
     });
   });
+
+  it('should report mutants that are ignored with an ignore plugin with the correct ignore reason', async () => {
+    const report = await readMutationTestingJsonResult();
+    const addResult = report.files['src/Add.js'];
+    const mutantsAtLine2 = addResult.mutants.filter(({ location }) => location.start.line === 2);
+    const mutantsAtLin8 = addResult.mutants.filter(({ location }) => location.start.line === 8);
+    const mutantsAtLin13 = addResult.mutants.filter(({ location }) => location.start.line === 13);
+    const mutantsAtLine18 = addResult.mutants.filter(({ location }) => location.start.line === 18);
+
+    expect(mutantsAtLine2).lengthOf(2);
+    expect(mutantsAtLin8).lengthOf(1);
+    expect(mutantsAtLin13).lengthOf(1);
+    expect(mutantsAtLine18).lengthOf(1);
+    [...mutantsAtLine2, ...mutantsAtLin8, ...mutantsAtLin13].forEach((mutant) => {
+      expect(mutant.status).eq(MutantStatus.Ignored);
+      expect(mutant.statusReason).eq("We're not interested in console.log statements for now");
+    });
+    mutantsAtLine18.forEach((mutant) => expect(mutant.status).eq(MutantStatus.NoCoverage));
+  });
 });

e2e/test/ignore-project/verify/verify.js.snap

@@ -3,20 +3,20 @@
 exports[`After running stryker on jest-react project should report expected scores 1`] = `
 Object {
   "compileErrors": 0,
-  "ignored": 28,
+  "ignored": 32,
   "killed": 8,
-  "mutationScore": 53.333333333333336,
+  "mutationScore": 50,
   "mutationScoreBasedOnCoveredCode": 100,
-  "noCoverage": 7,
+  "noCoverage": 8,
   "pending": 0,
   "runtimeErrors": 0,
   "survived": 0,
   "timeout": 0,
   "totalCovered": 8,
   "totalDetected": 8,
   "totalInvalid": 0,
-  "totalMutants": 43,
-  "totalUndetected": 7,
-  "totalValid": 15,
+  "totalMutants": 48,
+  "totalUndetected": 8,
+  "totalValid": 16,
 }
 `;

e2e/tsconfig.json

@@ -8,13 +8,13 @@
     "composite": false,
     "importHelpers": false,
     "strictNullChecks": false, // hard to handle without a non-null assertion
-    "useUnknownInCatchVariables": false,// hard to handle without type assertions
+    "useUnknownInCatchVariables": false, // hard to handle without type assertions
     "esModuleInterop": true,
     "rootDir": "..",
     "types": ["mocha", "node"],
     "skipLibCheck": true,
     "target": "ES2022",
     "lib": ["ES2022"]
   },
-  "include": ["tasks", "test/*/verify/*.js", "helpers.js"]
+  "include": ["tasks", "test/*/verify/*.js", "test/*/stryker-plugins/**/*.js", "test/*/stryker-plugins/**/*.ts", "helpers.js"]
 }

packages/api/package.json

@@ -19,6 +19,7 @@
   "exports": {
     "./check": "./dist/src/check/index.js",
     "./core": "./dist/src/core/index.js",
+    "./ignorer": "./dist/src/ignorer/index.js",
     "./logging": "./dist/src/logging/index.js",
     "./plugin": "./dist/src/plugin/index.js",
     "./report": "./dist/src/report/index.js",
@@ -62,6 +63,7 @@
     "typed-inject": "~4.0.0"
   },
   "devDependencies": {
+    "@babel/core": "~7.23.0",
     "@types/node": "18.18.5"
   }
 }

packages/api/schema/stryker-core.json

@@ -531,6 +531,14 @@
       "description": "Allows stryker to exit without any errors in cases where no tests are found",
       "type": "boolean",
       "default": false
+    },
+    "ignorers": {
+      "description": "Enable ignorer plugins here. An ignorer plugin will be invoked on each AST node visitation and can decide to ignore the node or not. This can be useful for example to ignore all mutations in a console.log() statement.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "default": []
     }
   }
 }

packages/api/src/ignorer/ignorer.ts

@@ -0,0 +1,8 @@
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface NodePath {
+  // Left empty so the declaration can be merged
+}
+
+export interface Ignorer {
+  shouldIgnore(path: NodePath): string | undefined;
+}

packages/api/src/ignorer/index.ts

@@ -0,0 +1 @@
+export * from './ignorer.js';

packages/api/src/plugin/plugin-kind.ts

@@ -5,4 +5,5 @@ export enum PluginKind {
   Checker = 'Checker',
   TestRunner = 'TestRunner',
   Reporter = 'Reporter',
+  Ignorer = 'Ignorer',
 }

packages/api/src/plugin/plugins.ts

@@ -4,6 +4,8 @@ import { Reporter } from '../report/index.js';
 import { TestRunner } from '../test-runner/index.js';
 import { Checker } from '../check/index.js';
 
+import { Ignorer } from '../ignorer/ignorer.js';
+
 import { PluginContext } from './contexts.js';
 import { PluginKind } from './plugin-kind.js';
 
@@ -82,6 +84,7 @@ export interface PluginInterfaces {
   [PluginKind.Reporter]: Reporter;
   [PluginKind.TestRunner]: TestRunner;
   [PluginKind.Checker]: Checker;
+  [PluginKind.Ignorer]: Ignorer;
 }
 
 /**