feat(ignore static): allow to ignore static mutants (#3284)

Add support for `--ignoreStatic`. When `ignoreStatic` is enabled, static mutants are ignored (defaults to `false`).

Static mutants can be expensive, depending on your use case. For example, 6% of mutants of `@stryker-mutator/core` are static, yet they take up > 50% of time executing tests. So it might make sense to ignore them.

docs/configuration.md

@@ -183,6 +183,28 @@ If a glob pattern starts with `/`, the pattern is relative to the current workin
 
 When using the command line, the list can only contain a comma separated list of globbing expressions.
 
+### `ignoreStatic` [`boolean`]
+
+Default: `false`<br />
+Command line: `--ignoreStatic`<br />
+Config file: `"ignoreStatic": true`<br />
+
+Static mutants are mutants which are only executed during the loading of a file. Testing these mutants come with a big performance penalty. Therefore, it might make sense to ignore static mutants altogether.
+
+For example:
+
+```js
+const hi = '👋'; // Mutant 👽 StringLiteral
+
+export function greet(name) {
+  return `${hi} ${name}`
+}
+```
+
+In this example, `'👋'` on line 1 would be mutated to an empty string by the StringLiteral mutator. However, the mutant is only executed _when the file is loaded_, making it a static mutant. It is impossible to measure the exact code coverage per test for the mutant. Therefore, Stryker would default to running all tests.
+
+_Note:_ Enabling `--ignoreStatic` requires `"coverageAnalysis": "perTest"`, because detecting which mutant is static is done during the initial test run and needs per test coverage analysis.
+
 ### `inPlace` [`boolean`]
 
 Default: `false`<br />

e2e/test/coverage-analysis/.mocharc.json

@@ -0,0 +1,4 @@
+{
+  "require": ["spec/chai-setup.js"],
+  "spec": ["spec/*.spec.js"]
+}

e2e/test/coverage-analysis/cucumber-features/concat.feature

@@ -5,3 +5,6 @@ Feature: concat
     When concat with 'bar'
     Then the result should be 'foobar'
 
+    Scenario: greet me
+    When I greet 'me'
+    Then the result should be '👋 me'

e2e/test/coverage-analysis/cucumber-features/step-definitions/general-steps.js

@@ -1,6 +1,6 @@
 const { Given, When, Then } = require('@cucumber/cucumber');
 const { expect } = require('chai');
-const { concat } = require('../../src/concat');
+const { concat, greet } = require('../../src/concat');
 const { add, multiply } = require('../../src/math');
 
 Given('input {string}', function (input) {
@@ -15,9 +15,14 @@ When('concat with {string}', function (other) {
   this.result = concat(this.input, other);
 });
 
+When('I greet {string}', function (subject) {
+  this.result = greet(subject);
+});
+
 When('add with {int}', function (other) {
   this.result = add(this.input, other);
 });
+
 When('multiplied with {int}', function (other) {
   this.result = multiply(this.input, other);
 });

e2e/test/coverage-analysis/jasmine.json

@@ -0,0 +1,5 @@
+{
+  "spec_files": [
+    "spec/*.spec.js"
+  ]
+}

e2e/test/coverage-analysis/jest.config.json

@@ -0,0 +1,4 @@
+{
+  "testEnvironment": "node",
+  "testMatch": ["<rootDir>/spec/*.spec.js"]
+}

e2e/test/coverage-analysis/karma.conf.js

@@ -0,0 +1,21 @@
+// Karma configuration
+// Generated on Tue Nov 30 2021 09:57:14 GMT+0100 (Central European Standard Time)
+
+module.exports = function(config) {
+  config.set({
+    basePath: '',
+    frameworks: ['chai', 'jasmine'],
+    files: [
+      'src/**/*.js',
+      'spec/**/*.js'
+    ],
+    reporters: ['progress'],
+    port: 9876,
+    colors: true,
+    logLevel: config.LOG_INFO,
+    autoWatch: false,
+    browsers: ['ChromeHeadless'],
+    singleRun: true,
+    concurrency: Infinity
+  })
+}

e2e/test/coverage-analysis/package.json

@@ -2,7 +2,10 @@
   "name": "coverage-analysis",
   "description": "A e2e test for --coverageAnalysis",
   "scripts": {
-    "test:unit": "jasmine",
+    "test:unit:cucumber": "cucumber-js",
+    "test:unit:jasmine": "jasmine",
+    "test:unit:mocha": "mocha",
+    "test:unit:karma": "karma start",
     "test": "mocha --no-package --timeout 60000 --require \"../../tasks/ts-node-register.js\" verify/verify.ts"
   }
 }

e2e/test/coverage-analysis/spec/chai-setup.js

@@ -0,0 +1,13 @@
+{
+  const chai = globalThis.chai ?? require('chai');
+
+  globalThis.expect = chai.expect;
+  chai.util.addMethod(chai.Assertion.prototype, 'toEqual', function (expected) {
+    var obj = chai.util.flag(this, 'object');
+    new chai.Assertion(obj).to.deep.equal(expected);
+  });
+  chai.util.addMethod(chai.Assertion.prototype, 'toBe', function (expected) {
+    var obj = chai.util.flag(this, 'object');
+    new chai.Assertion(obj).to.equal(expected);
+  });
+}

e2e/test/coverage-analysis/spec/concat.spec.js

@@ -0,0 +1,16 @@
+{
+  const concat = globalThis.concat ?? require('../src/concat').concat;
+  const greet = globalThis.greet ?? require('../src/concat').greet;
+
+  describe('concat', () => {
+    it('should concat a and b', () => {
+      expect(concat('foo', 'bar')).toBe('foobar');
+    });
+  });
+
+  describe('greet', () => {
+    it('should greet me', () => {
+      expect(greet('me')).toBe('👋 me')
+    });
+  });
+}

e2e/test/coverage-analysis/spec/math.spec.js

@@ -0,0 +1,20 @@
+{
+
+  const add = globalThis.add ?? require('../src/math').add;
+  const multiply = globalThis.multiply ?? require('../src/math').multiply;
+
+  describe('add', () => {
+    it('should add two numbers', () => {
+      expect(add(1, 2)).toBe(3);
+    });
+  });
+
+  describe('multiply', () => {
+    it('should multiply the numbers', () => {
+      // Bad test, surviving mutant when * => /
+      expect(multiply(2, 1)).toBe(2);
+    });
+  });
+
+  // Missing describe for `addOne` -> surviving / noCoverage mutant
+}

e2e/test/coverage-analysis/src/concat.js

@@ -1,3 +1,18 @@
-module.exports.concat = function concat(a, b){
+function concat(a, b){
   return `${a}${b}`;
 };
+
+// Static mutant
+const hi = '👋';
+
+function greet(name) {
+  return `${hi} ${name}`
+}
+
+// Stryker disable all: Not useful for coverage analysis test
+if(typeof module === 'object') {
+  module.exports = {
+    concat,
+    greet
+  }
+}

e2e/test/coverage-analysis/src/math.js

@@ -10,8 +10,11 @@ function addOne(a) {
   return ++a;
 }
 
-module.exports = {
-  add,
-  multiply,
-  addOne
-};
+// Stryker disable all: Not useful for coverage analysis test
+if (typeof module === 'object') {
+  module.exports = {
+    add,
+    multiply,
+    addOne,
+  };
+}