feat(hit limit): infinite loop prevention in mocha-runner

Add infinite loop prevention using a hit counter to `@stryker-mutator/mocha-runner`.

Co-authored-by: Nico Jansen <jansennico@gmail.com>

e2e/test/hit-limit/.mocharc.json

@@ -0,0 +1,3 @@
+{
+  "spec": "test/**.mocha.spec.js"
+}

e2e/test/hit-limit/package.json

@@ -5,6 +5,7 @@
   "description": "A e2e test project to verify the infinite loop prevention behavior in different test runner (hit limit)",
   "main": "index.js",
   "scripts": {
+    "test:mocha": "mocha",
     "test:karma": "karma start",
     "test": "mocha --no-package --timeout 60000 --require \"../../tasks/ts-node-register.js\" verify/verify.ts"
   },

e2e/test/hit-limit/test/loop.mocha.spec.js

@@ -0,0 +1,10 @@
+const { loop } = require('../src/loop');
+const { expect } = require('chai');
+
+describe('loop', () => {
+  it('should result in 15 for n=5 and a sum function', () => {
+    let result = 0;
+    loop(5, (n) => (result += n));
+    expect(result).to.eq(15);
+  });
+});

e2e/test/hit-limit/verify/verify.ts

@@ -3,11 +3,18 @@ import { Stryker } from '@stryker-mutator/core';
 import { MutantStatus } from 'mutation-testing-report-schema/api';
 
 describe('Limit counter', () => {
-
   it('should limit infinite loops in the karma-runner', async () => {
     const stryker = new Stryker({ testRunner: 'karma' });
     const results = await stryker.runMutationTest();
-    const timeoutResults = results.filter(res => res.status === MutantStatus.Timeout);
+    const timeoutResults = results.filter((res) => res.status === MutantStatus.Timeout);
+    expect(timeoutResults).lengthOf(3);
+    timeoutResults.forEach((result) => expect(result.statusReason).eq('Hit limit reached (501/500)'));
+  });
+
+  it('should limit infinite loops in the mocha-runner', async () => {
+    const stryker = new Stryker({ testRunner: 'mocha' });
+    const results = await stryker.runMutationTest();
+    const timeoutResults = results.filter((res) => res.status === MutantStatus.Timeout);
     expect(timeoutResults).lengthOf(3);
     timeoutResults.forEach((result) => expect(result.statusReason).eq('Hit limit reached (501/500)'));
   });

packages/mocha-runner/src/mocha-test-runner.ts

@@ -12,6 +12,7 @@ import {
   DryRunStatus,
   toMutantRunResult,
   CompleteDryRunResult,
+  determineHitLimitReached,
 } from '@stryker-mutator/api/test-runner';
 
 import { MochaOptions } from '../src-generated/mocha-runner-options';
@@ -78,8 +79,10 @@ export class MochaTestRunner implements TestRunner {
     return runResult;
   }
 
-  public async mutantRun({ activeMutant, testFilter, disableBail }: MutantRunOptions): Promise<MutantRunResult> {
+  public async mutantRun({ activeMutant, testFilter, disableBail, hitLimit }: MutantRunOptions): Promise<MutantRunResult> {
     this.instrumenterContext.activeMutant = activeMutant.id;
+    this.instrumenterContext.hitLimit = hitLimit;
+    this.instrumenterContext.hitCount = hitLimit ? 0 : undefined;
     // eslint-disable-next-line @typescript-eslint/no-empty-function
     let intercept: (mocha: Mocha) => void = () => {};
     if (testFilter) {
@@ -115,6 +118,10 @@ export class MochaTestRunner implements TestRunner {
       }
       const reporter = StrykerMochaReporter.currentInstance;
       if (reporter) {
+        const timeoutResult = determineHitLimitReached(this.instrumenterContext.hitCount, this.instrumenterContext.hitLimit);
+        if (timeoutResult) {
+          return timeoutResult;
+        }
         const result: CompleteDryRunResult = {
           status: DryRunStatus.Complete,
           tests: reporter.tests,

packages/mocha-runner/test/integration/timeout-on-infinite-loop.it.spec.ts

@@ -0,0 +1,58 @@
+import { testInjector, factory, assertions } from '@stryker-mutator/test-helpers';
+import { expect } from 'chai';
+
+import { createMochaOptions } from '../helpers/factories';
+import { createMochaTestRunnerFactory, MochaTestRunner } from '../../src';
+import { resolveTestResource } from '../helpers/resolve-test-resource';
+
+describe('Infinite loop', () => {
+  let sut: MochaTestRunner;
+
+  beforeEach(async () => {
+    const spec = [
+      resolveTestResource('infinite-loop-instrumented', 'infinite-loop.spec.js'),
+      resolveTestResource('infinite-loop', 'infinite-loop.spec.js'),
+    ];
+    testInjector.options.mochaOptions = createMochaOptions({ spec });
+    sut = testInjector.injector.injectFunction(createMochaTestRunnerFactory('__stryker2__'));
+    await sut.init();
+  });
+
+  it('should be able to recover using a hit counter', async () => {
+    // Arrange
+    const options = factory.mutantRunOptions({
+      activeMutant: factory.mutant({ id: '20' }),
+      testFilter: ['should be able to break out of an infinite loop with a hit counter'],
+      hitLimit: 10,
+    });
+
+    // Act
+    const result = await sut.mutantRun(options);
+
+    // Assert
+    assertions.expectTimeout(result);
+    expect(result.reason).contains('Hit limit reached');
+  });
+
+  it('should reset hit counter state correctly between runs', async () => {
+    const firstResult = await sut.mutantRun(
+      factory.mutantRunOptions({
+        activeMutant: factory.mutant({ id: '20' }),
+        testFilter: ['should be able to break out of an infinite loop with a hit counter'],
+        hitLimit: 10,
+      })
+    );
+    const secondResult = await sut.mutantRun(
+      factory.mutantRunOptions({
+        // 27 is a 'normal' mutant that should be killed
+        activeMutant: factory.mutant({ id: '23' }),
+        testFilter: ['should be able to break out of an infinite loop with a hit counter'],
+        hitLimit: 10,
+      })
+    );
+
+    // Assert
+    assertions.expectTimeout(firstResult);
+    assertions.expectKilled(secondResult);
+  });
+});

packages/mocha-runner/test/setup.ts

@@ -20,6 +20,8 @@ export const mochaHooks = {
     sinon.restore();
     testInjector.reset();
     StrykerMochaReporter.currentInstance = undefined;
-    delete global.__stryker2__;
+    delete global.__stryker2__?.activeMutant;
+    delete global.__stryker2__?.currentTestId;
+    delete global.__stryker2__?.mutantCoverage;
   },
 };

packages/mocha-runner/test/unit/mocha-test-runner.spec.ts

@@ -295,9 +295,27 @@ describe(MochaTestRunner.name, () => {
       assertions.expectSurvived(result);
     });
 
-    async function actMutantRun(options = factory.mutantRunOptions()) {
+    it('should report a timeout when the hitLimit was reached', async () => {
+      reporterMock.tests = [factory.failedTestResult()];
+      const result = await actMutantRun(factory.mutantRunOptions({ hitLimit: 9 }), 10);
+      assertions.expectTimeout(result);
+      expect(result.reason).contains('Hit limit reached (10/9)');
+    });
+
+    it('should reset the hitLimit between runs', async () => {
+      reporterMock.tests = [factory.failedTestResult()];
+      const firstResult = await actMutantRun(factory.mutantRunOptions({ hitLimit: 9 }), 10);
+      reporterMock.tests = [factory.failedTestResult()];
+      const secondResult = await actMutantRun(factory.mutantRunOptions({ hitLimit: undefined }), 10);
+      assertions.expectTimeout(firstResult);
+      assertions.expectKilled(secondResult);
+    });
+
+    async function actMutantRun(options = factory.mutantRunOptions(), hitCount?: number) {
       mocha.run.callsArg(0);
-      return sut.mutantRun(options);
+      const result = sut.mutantRun(options);
+      global.__stryker2__!.hitCount = hitCount;
+      return result;
     }
   });
 });

packages/mocha-runner/testResources/infinite-loop-instrumented/infinite-loop.js

@@ -0,0 +1,84 @@
+// This file is generated with tasks/instrument-test-resources.js
+ function stryNS_9fa48() {
+  var g = new Function("return this")();
+  var ns = g.__stryker2__ || (g.__stryker2__ = {});
+
+  if (ns.activeMutant === undefined && g.process && g.process.env && g.process.env.__STRYKER_ACTIVE_MUTANT__) {
+    ns.activeMutant = g.process.env.__STRYKER_ACTIVE_MUTANT__;
+  }
+
+  function retrieveNS() {
+    return ns;
+  }
+
+  stryNS_9fa48 = retrieveNS;
+  return retrieveNS();
+}
+
+stryNS_9fa48();
+
+function stryCov_9fa48() {
+  var ns = stryNS_9fa48();
+  var cov = ns.mutantCoverage || (ns.mutantCoverage = {
+    static: {},
+    perTest: {}
+  });
+
+  function cover() {
+    var c = cov.static;
+
+    if (ns.currentTestId) {
+      c = cov.perTest[ns.currentTestId] = cov.perTest[ns.currentTestId] || {};
+    }
+
+    var a = arguments;
+
+    for (var i = 0; i < a.length; i++) {
+      c[a[i]] = (c[a[i]] || 0) + 1;
+    }
+  }
+
+  stryCov_9fa48 = cover;
+  cover.apply(null, arguments);
+}
+
+function stryMutAct_9fa48(id) {
+  var ns = stryNS_9fa48();
+
+  function isActive(id) {
+    if (ns.activeMutant === id) {
+      if (ns.hitCount !== void 0 && ++ns.hitCount > ns.hitLimit) {
+        throw new Error('Stryker: Hit count limit reached (' + ns.hitCount + ')');
+      }
+
+      return true;
+    }
+
+    return false;
+  }
+
+  stryMutAct_9fa48 = isActive;
+  return isActive(id);
+}
+
+function loop(n, action) {
+  if (stryMutAct_9fa48("16")) {
+    {}
+  } else {
+    stryCov_9fa48("16");
+    let goOn = stryMutAct_9fa48("17") ? false : (stryCov_9fa48("17"), true);
+
+    while (stryMutAct_9fa48("18") ? false : (stryCov_9fa48("18"), goOn)) {
+      if (stryMutAct_9fa48("19")) {
+        {}
+      } else {
+        stryCov_9fa48("19");
+        action(n);
+        stryMutAct_9fa48("20") ? n++ : (stryCov_9fa48("20"), n--);
+        goOn = stryMutAct_9fa48("24") ? n <= 0 : stryMutAct_9fa48("23") ? n >= 0 : stryMutAct_9fa48("22") ? false : stryMutAct_9fa48("21") ? true : (stryCov_9fa48("21", "22", "23", "24"), n > 0);
+      }
+    }
+  }
+}
+
+module.exports = loop;

packages/mocha-runner/testResources/infinite-loop-instrumented/infinite-loop.spec.js

@@ -0,0 +1,17 @@
+var expect = require('chai').expect;
+var loop = require('./infinite-loop');
+
+it('should handle an infinite loop as a timeout', () => {
+  while (true);
+});
+
+it('should be able to recover and test others', () => {});
+
+it('should be able to break out of an infinite loop with a hit counter', () => {
+  let total = 0;
+  loop(5, (n) => {
+    expect(n).not.eq(0);
+    total += n;
+  });
+  expect(total).eq(15);
+});

packages/mocha-runner/testResources/infinite-loop/infinite-loop.js

@@ -0,0 +1,10 @@
+function loop(n, action) {
+  let goOn = true;
+  while (goOn) {
+    action(n);
+    n--;
+    goOn = n > 0;
+  }
+}
+
+module.exports = loop;

packages/mocha-runner/testResources/infinite-loop/infinite-loop.spec.js

@@ -0,0 +1,17 @@
+var expect = require('chai').expect;
+var loop = require('./infinite-loop');
+
+it('should handle an infinite loop as a timeout', () => {
+  while (true);
+});
+
+it('should be able to recover and test others', () => {});
+
+it('should be able to break out of an infinite loop with a hit counter', () => {
+  let total = 0;
+  loop(5, (n) => {
+    expect(n).not.eq(0);
+    total += n;
+  });
+  expect(total).eq(15);
+});

tasks/instrument-test-resources.js

@@ -12,7 +12,8 @@ const instrumenter = new Instrumenter({
 
 async function main() {
   await instrument({
-    './packages/mocha-runner/testResources/sample-project/MyMath.js': './packages/mocha-runner/testResources/sample-project-instrumented/MyMath.js'
+    './packages/mocha-runner/testResources/sample-project/MyMath.js': './packages/mocha-runner/testResources/sample-project-instrumented/MyMath.js',
+    './packages/mocha-runner/testResources/infinite-loop/infinite-loop.js': './packages/mocha-runner/testResources/infinite-loop-instrumented/infinite-loop.js',
   }, '__stryker2__');
   await instrument({
     './packages/jasmine-runner/testResources/jasmine-init/lib/jasmine_examples/Player.js': './packages/jasmine-runner/testResources/jasmine-init-instrumented/lib/jasmine_examples/Player.js',