feat(karma-runner): disable client.clearContext (#2048)

Disable client.clearContext to avoid a race condition in Karma where a mutant is unfairly reported as "RuntimeError".

e2e/tasks/run-e2e-tests.ts

@@ -26,21 +26,31 @@ function runE2eTests() {
 
 runE2eTests()
   .subscribe({
-    complete: () => console.log('Done'),
+    complete: () => console.log('✅ Done'),
     error: err => {
       console.error(err);
       process.exit(1);
     },
   });
 
-function execNpm(command: string, testDir: string) {
+function execNpm(command: string, testDir: string, stream: boolean) {
   const currentTestDir = path.resolve(testRootDir, testDir);
   console.log(`Exec ${testDir} npm ${command}`);
   const testProcess = execa('npm', [command], { timeout: 500000, cwd: currentTestDir, stdio: 'pipe' });
   let stderr = '';
   let stdout = '';
-  testProcess.stderr.on('data', chunk => stderr += chunk.toString());
-  testProcess.stdout.on('data', chunk => stdout += chunk.toString());
+  testProcess.stderr.on('data', chunk => {
+    stderr += chunk.toString();
+    if (stream) {
+      console.error(chunk.toString());
+    }
+  });
+  testProcess.stdout.on('data', chunk => {
+    stdout += chunk.toString()
+    if (stream) {
+      console.log(chunk.toString());
+    }
+  });
   return testProcess.catch(error => {
     console.log(`X ${testDir}`);
     console.log(stdout);
@@ -62,7 +72,7 @@ function satisfiesNodeVersion(testDir: string): boolean {
 
 async function runTest(testDir: string) {
   if (satisfiesNodeVersion(testDir)) {
-    await execNpm('test', testDir);
+    await execNpm('test', testDir, false);
   }
   return testDir;
 }

e2e/test/karma-jasmine/stryker.conf.js

@@ -13,6 +13,7 @@ module.exports = function (config) {
         }
       }
     },
+    timeoutMS: 60000,
     mutator: 'javascript'
   });
 };

e2e/test/karma-mocha/stryker.conf.js

@@ -8,12 +8,10 @@ module.exports = function (config) {
     karma: {
       config: {
         frameworks: ['mocha', 'chai'],
-        files: ['src/*.js', 'test/*.js'],
-        client: {
-          clearContext: false
-        }
+        files: ['src/*.js', 'test/*.js']
       }
     },
+    timeoutMS: 60000,
     maxConcurrentTestRunners: 2,
     coverageAnalysis: 'perTest'
   });

packages/karma-runner/src/starters/stryker-karma.conf.ts

@@ -51,6 +51,19 @@ function setLifeCycleOptions(config: Config) {
   });
 }
 
+/**
+ * Sets configuration that is needed to control client scripts in karma.
+ * @param config The config to use
+ * @see https://github.com/stryker-mutator/stryker/issues/2049
+ */
+function setClientOptions(config: Config) {
+  // Disable clearContext because of issue #2049 (race condition in Karma)
+  // Enabling clearContext (default true) will load "about:blank" in the iFrame after a test run.
+  // As far as I can see clearing the context only has a visible effect (you don't see the result of the last test).
+  // If this is true, disabling it is safe to do and solves the race condition issue.
+  config.set({ client: { clearContext: false } });
+}
+
 function setUserKarmaConfig(config: Config) {
   if (globalSettings.karmaConfig) {
     config.set(globalSettings.karmaConfig);
@@ -153,6 +166,7 @@ export = Object.assign(
     setUserKarmaConfig(config);
     setBasePath(config);
     setLifeCycleOptions(config);
+    setClientOptions(config);
     configureTestHooksMiddleware(config);
     configureStrykerReporter(config);
   },

packages/karma-runner/test/integration/KarmaTestRunner.it.spec.ts

@@ -78,10 +78,9 @@ describe(`${KarmaTestRunner.name} integration`, () => {
 
     it('should report completed tests', async () => {
       const runResult = await sut.run({});
-      // await new Promise(res => {});
       expectToHaveSuccessfulTests(runResult, 5);
       expectToHaveFailedTests(runResult, []);
-      expect(runResult.status).to.be.eq(RunStatus.Complete);
+      expect(RunStatus[runResult.status]).to.be.eq(RunStatus[RunStatus.Complete]);
     });
 
     it('should be able to filter tests', async () => {
@@ -111,7 +110,7 @@ describe(`${KarmaTestRunner.name} integration`, () => {
         return expect(sut.run({})).to.eventually.satisfy((runResult: RunResult) => {
           expectToHaveSuccessfulTests(runResult, 5);
           expectToHaveFailedTests(runResult, []);
-          expect(runResult.status).to.be.eq(RunStatus.Complete);
+          expect(RunStatus[runResult.status]).to.be.eq(RunStatus[RunStatus.Complete]);
           return true;
         });
       });

packages/karma-runner/test/unit/starters/stryker-karma.conf.spec.ts

@@ -95,6 +95,18 @@ describe('stryker-karma.conf.js', () => {
     });
   });
 
+  // See https://github.com/stryker-mutator/stryker/issues/2049
+  it('should force clearContext to false', () => {
+    // Arrange
+    requireModuleStub.returns((conf: Config) => conf.set({ client: { clearContext: true } }));
+
+    // Act
+    sut(config);
+
+    // Assert
+    expect(config).deep.include({ client: { clearContext: false } });
+  });
+
   it('should configure the tests hooks middleware', () => {
     sut(config);
     expect(config).deep.include({

packages/karma-runner/testResources/sampleProject/test-mocha/AddSpec.js

@@ -1,5 +1,5 @@
 describe('Add', function () {
-  it('should be able to add two numbers', function () {
+  it('should be able to add two numbers', function (done) {
     var num1 = 2;
     var num2 = 5;
     var expected = num1 + num2;