feat(jest-runner): support mutation switching

packages/jest-runner/.vscode/launch.json

@@ -20,7 +20,10 @@
       "outFiles": [
         "${workspaceRoot}/test/**/*.js",
         "${workspaceRoot}/src/**/*.js"
-      ]
+      ],
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
     },
     {
       "type": "node",
@@ -38,7 +41,10 @@
       "outFiles": [
         "${workspaceRoot}/test/**/*.js",
         "${workspaceRoot}/src/**/*.js"
-      ]
+      ],
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
     }
   ]
-}
+}

packages/jest-runner/package.json

@@ -4,8 +4,8 @@
   "description": "A plugin to use the jest test runner and framework in Stryker, the JavaScript mutation testing framework",
   "main": "src/index.js",
   "scripts": {
-    "test": "nyc --exclude-after-remap=false --check-coverage --reporter=html --report-dir=reports/coverage --lines 80 --functions 80 --branches 70 npm run mocha",
-    "mocha": "mocha \"test/helpers/**/*.js\" \"test/unit/**/*.js\" && mocha --timeout 30000 \"test/helpers/**/*.js\" \"test/integration/**/*.js\"",
+    "test": "nyc --exclude-after-remap=false --check-coverage --reporter=html --report-dir=reports/coverage --lines 80 --functions 80 --branches 75 npm run mocha",
+    "mocha": "mocha --timeout 30000 \"test/helpers/**/*.js\" \"test/integration/**/*.js\"",
     "stryker": "node ../core/bin/stryker run"
   },
   "repository": {
@@ -38,6 +38,7 @@
   },
   "homepage": "https://github.com/stryker-mutator/stryker/tree/master/packages/jest-runner#readme",
   "devDependencies": {
+    "@jest/types": "^26.2.0",
     "@stryker-mutator/test-helpers": "4.0.0-beta.1",
     "@types/node": "^14.0.1",
     "@types/semver": "~7.3.1",
@@ -53,6 +54,7 @@
   },
   "dependencies": {
     "@stryker-mutator/api": "4.0.0-beta.1",
+    "@stryker-mutator/util": "4.0.0-beta.1",
     "semver": "~6.3.0"
   },
   "initStrykerConfig": {

packages/jest-runner/src/JestRunResult.ts

@@ -0,0 +1,7 @@
+import type { Config } from '@jest/types';
+import type { AggregatedResult } from '@jest/test-result';
+
+export interface JestRunResult {
+  results: AggregatedResult;
+  globalConfig: Config.GlobalConfig;
+}

packages/jest-runner/src/JestTestRunner.ts

@@ -1,13 +1,30 @@
-import { StrykerOptions } from '@stryker-mutator/api/core';
+import { StrykerOptions, INSTRUMENTER_CONSTANTS, Mutant } from '@stryker-mutator/api/core';
 import { Logger } from '@stryker-mutator/api/logging';
 import { commonTokens, Injector, OptionsContext, tokens } from '@stryker-mutator/api/plugin';
-import { RunOptions, RunResult, RunStatus, TestResult, TestRunner, TestStatus } from '@stryker-mutator/api/test_runner';
+import {
+  TestRunner2,
+  DryRunOptions,
+  MutantRunOptions,
+  DryRunResult,
+  MutantRunResult,
+  toMutantRunResult,
+  DryRunStatus,
+  TestResult,
+  TestStatus,
+} from '@stryker-mutator/api/test_runner2';
+import { notEmpty } from '@stryker-mutator/util';
+import type * as jest from '@jest/types';
+import type * as jestTestResult from '@jest/test-result';
+
+import { SerializableError } from '@jest/types/build/TestResult';
 
 import { jestTestAdapterFactory } from './jestTestAdapters';
 import JestTestAdapter from './jestTestAdapters/JestTestAdapter';
 import JestConfigLoader from './configLoaders/JestConfigLoader';
 import { configLoaderToken, processEnvToken, jestTestAdapterToken, jestVersionToken } from './pluginTokens';
 import { configLoaderFactory } from './configLoaders';
+import { JestRunnerOptionsWithStrykerOptions } from './JestRunnerOptionsWithStrykerOptions';
+import JEST_OVERRIDE_OPTIONS from './jestOverrideOptions';
 
 export function jestTestRunnerFactory(injector: Injector<OptionsContext>) {
   return injector
@@ -19,8 +36,9 @@ export function jestTestRunnerFactory(injector: Injector<OptionsContext>) {
 }
 jestTestRunnerFactory.inject = tokens(commonTokens.injector);
 
-export default class JestTestRunner implements TestRunner {
-  private readonly jestConfig: Jest.Configuration;
+export default class JestTestRunner implements TestRunner2 {
+  private readonly jestConfig: jest.Config.InitialOptions;
+  private mutantRunJestConfigCache: jest.Config.InitialOptions | undefined;
 
   private readonly enableFindRelatedTests: boolean;
 
@@ -32,100 +50,153 @@ export default class JestTestRunner implements TestRunner {
     private readonly jestTestAdapter: JestTestAdapter,
     configLoader: JestConfigLoader
   ) {
-    const errorMessage =
-      'This version of Stryker does not (yet) support Jest, sorry! Follow https://github.com/stryker-mutator/stryker/issues/2321 for the latest status.';
-    this.log.error(errorMessage);
-    throw new Error(errorMessage);
-
-    // const jestOptions = options as JestRunnerOptionsWithStrykerOptions;
-    // // Get jest configuration from stryker options and assign it to jestConfig
-    // const configFromFile = configLoader.loadConfig();
-    // this.jestConfig = this.mergeConfigSettings(configFromFile, (jestOptions.jest.config as any) || {});
-
-    // // Get enableFindRelatedTests from stryker jest options or default to true
-    // this.enableFindRelatedTests = jestOptions.jest.enableFindRelatedTests;
-    // if (this.enableFindRelatedTests === undefined) {
-    //   this.enableFindRelatedTests = true;
-    // }
-
-    // if (this.enableFindRelatedTests) {
-    //   this.log.debug('Running jest with --findRelatedTests flag. Set jest.enableFindRelatedTests to false to run all tests on every mutant.');
-    // } else {
-    //   this.log.debug(
-    //     'Running jest without --findRelatedTests flag. Set jest.enableFindRelatedTests to true to run only relevant tests on every mutant.'
-    //   );
-    // }
-
-    // // basePath will be used in future releases of Stryker as a way to define the project root
-    // // Default to process.cwd when basePath is not set for now, should be removed when issue is solved
-    // // https://github.com/stryker-mutator/stryker/issues/650
-    // this.jestConfig.rootDir = (options.basePath as string) || process.cwd();
-    // this.log.debug(`Project root is ${this.jestConfig.rootDir}`);
+    const jestOptions = options as JestRunnerOptionsWithStrykerOptions;
+    // Get jest configuration from stryker options and assign it to jestConfig
+    const configFromFile = configLoader.loadConfig();
+    this.jestConfig = this.mergeConfigSettings(configFromFile, (jestOptions.jest.config as any) || {});
+
+    // Get enableFindRelatedTests from stryker jest options or default to true
+    this.enableFindRelatedTests = jestOptions.jest.enableFindRelatedTests;
+    if (this.enableFindRelatedTests === undefined) {
+      this.enableFindRelatedTests = true;
+    }
+
+    if (this.enableFindRelatedTests) {
+      this.log.debug('Running jest with --findRelatedTests flag. Set jest.enableFindRelatedTests to false to run all tests on every mutant.');
+    } else {
+      this.log.debug(
+        'Running jest without --findRelatedTests flag. Set jest.enableFindRelatedTests to true to run only relevant tests on every mutant.'
+      );
+    }
+
+    // basePath will be used in future releases of Stryker as a way to define the project root
+    // Default to process.cwd when basePath is not set for now, should be removed when issue is solved
+    // https://github.com/stryker-mutator/stryker/issues/650
+    this.jestConfig.rootDir = (options.basePath as string) || process.cwd();
+    this.log.debug(`Project root is ${this.jestConfig.rootDir}`);
   }
 
-  public async run(options: RunOptions): Promise<RunResult> {
-    this.setNodeEnv();
-    const { results } = await this.jestTestAdapter.run(
-      this.jestConfig,
-      process.cwd(),
-      this.enableFindRelatedTests ? options.mutatedFileName : undefined
-    );
+  public dryRun(_: DryRunOptions): Promise<DryRunResult> {
+    return this.run(this.jestConfig);
+  }
+  public async mutantRun({ activeMutant }: MutantRunOptions): Promise<MutantRunResult> {
+    const fileUnderTest = this.enableFindRelatedTests ? activeMutant.fileName : undefined;
+    const dryRunResult = await this.run(this.getMutantRunOptions(activeMutant), fileUnderTest);
+    return toMutantRunResult(dryRunResult);
+  }
 
-    // Get the non-empty errorMessages from the jest RunResult, it's safe to cast to Array<string> here because we filter the empty error messages
-    const errorMessages = results.testResults
-      .map((testSuite: Jest.TestResult) => testSuite.failureMessage)
-      .filter((errorMessage) => errorMessage) as string[];
+  private getMutantRunOptions(activeMutant: Mutant): jest.Config.InitialOptions {
+    if (!this.mutantRunJestConfigCache) {
+      const extraGlobals: string[] = [INSTRUMENTER_CONSTANTS.ACTIVE_MUTANT];
+      if (this.jestConfig.extraGlobals) {
+        extraGlobals.push(...this.jestConfig.extraGlobals);
+      }
 
-    return {
-      errorMessages,
-      status: results.numRuntimeErrorTestSuites > 0 ? RunStatus.Error : RunStatus.Complete,
-      tests: this.processTestResults(results.testResults),
-    };
+      this.mutantRunJestConfigCache = {
+        ...this.jestConfig,
+        globals: {
+          ...this.jestConfig.globals,
+        },
+        extraGlobals,
+      };
+    }
+    this.mutantRunJestConfigCache.globals![INSTRUMENTER_CONSTANTS.ACTIVE_MUTANT] = activeMutant.id;
+    return this.mutantRunJestConfigCache;
   }
 
-  private setNodeEnv() {
+  private async run(config: jest.Config.InitialOptions, fileUnderTest: string | undefined = undefined): Promise<DryRunResult> {
+    this.setEnv();
+    const all = await this.jestTestAdapter.run(config, process.cwd(), fileUnderTest);
+
+    return this.collectRunResult(all.results);
+  }
+
+  private collectRunResult(results: jestTestResult.AggregatedResult): DryRunResult {
+    if (results.numRuntimeErrorTestSuites) {
+      const errorMessage = results.testResults
+        .map((testSuite) => this.collectSerializableErrorText(testSuite.testExecError))
+        .filter(notEmpty)
+        .join(', ');
+      return {
+        status: DryRunStatus.Error,
+        errorMessage,
+      };
+    } else {
+      return {
+        status: DryRunStatus.Complete,
+        tests: this.processTestResults(results.testResults),
+      };
+    }
+  }
+
+  private collectSerializableErrorText(error: SerializableError | undefined): string | undefined {
+    return error && `${error.code && `${error.code} `}${error.message} ${error.stack}`;
+  }
+
+  private setEnv() {
     // Jest CLI will set process.env.NODE_ENV to 'test' when it's null, do the same here
     // https://github.com/facebook/jest/blob/master/packages/jest-cli/bin/jest.js#L12-L14
     if (!this.processEnvRef.NODE_ENV) {
       this.processEnvRef.NODE_ENV = 'test';
     }
+
+    // Force colors off: https://github.com/chalk/supports-color#info
+    process.env.FORCE_COLOR = '0';
   }
 
-  private processTestResults(suiteResults: Jest.TestResult[]): TestResult[] {
+  private processTestResults(suiteResults: jestTestResult.TestResult[]): TestResult[] {
     const testResults: TestResult[] = [];
 
     for (const suiteResult of suiteResults) {
       for (const testResult of suiteResult.testResults) {
-        testResults.push({
-          failureMessages: testResult.failureMessages,
-          name: testResult.fullName,
-          status: this.determineTestResultStatus(testResult.status),
-          timeSpentMs: testResult.duration ? testResult.duration : 0,
-        });
+        let result: TestResult;
+        let timeSpentMs = testResult.duration ?? 0;
+
+        switch (testResult.status) {
+          case 'passed':
+            result = {
+              id: testResult.fullName,
+              name: testResult.fullName,
+              status: TestStatus.Success,
+              timeSpentMs,
+            };
+            break;
+          case 'failed':
+            result = {
+              id: testResult.fullName,
+              name: testResult.fullName,
+              failureMessage: testResult.failureMessages?.join(', '),
+              status: TestStatus.Failed,
+              timeSpentMs,
+            };
+            break;
+          default:
+            result = {
+              id: testResult.fullName,
+              name: testResult.fullName,
+              status: TestStatus.Skipped,
+              timeSpentMs,
+            };
+            break;
+        }
+        testResults.push(result);
       }
     }
 
     return testResults;
   }
 
-  private determineTestResultStatus(status: Jest.Status) {
-    switch (status) {
-      case 'passed':
-        return TestStatus.Success;
-      case 'failed':
-        return TestStatus.Failed;
-      default:
-        return TestStatus.Skipped;
-    }
+  private mergeConfigSettings(configFromFile: jest.Config.InitialOptions, config: jest.Config.InitialOptions): jest.Config.InitialOptions {
+    const stringify = (obj: object) => JSON.stringify(obj, null, 2);
+    this.log.debug(
+      `Merging file-based config ${stringify(configFromFile)} 
+      with custom config ${stringify(config)}
+      and default (internal) stryker config ${JEST_OVERRIDE_OPTIONS}`
+    );
+    return {
+      ...configFromFile,
+      ...config,
+      ...JEST_OVERRIDE_OPTIONS,
+    };
   }
-
-  // private mergeConfigSettings(configFromFile: Jest.Configuration, config: Jest.Configuration) {
-  //   const stringify = (obj: Record<string, any>) => JSON.stringify(obj, null, 2);
-  //   this.log.trace(
-  //     `Merging file-based config ${stringify(configFromFile)}
-  //     with custom config ${stringify(config)}
-  //     and default (internal) stryker config ${JEST_OVERRIDE_OPTIONS}`
-  //   );
-  //   return Object.assign(configFromFile, config, JEST_OVERRIDE_OPTIONS);
-  // }
 }

packages/jest-runner/src/configLoaders/CustomJestConfigLoader.ts

@@ -4,6 +4,7 @@ import path from 'path';
 import { Logger } from '@stryker-mutator/api/logging';
 import { tokens, commonTokens } from '@stryker-mutator/api/plugin';
 import { StrykerOptions } from '@stryker-mutator/api/core';
+import { Config } from '@jest/types';
 
 import { loaderToken, projectRootToken } from '../pluginTokens';
 import { JestRunnerOptionsWithStrykerOptions } from '../JestRunnerOptionsWithStrykerOptions';
@@ -24,7 +25,7 @@ export default class CustomJestConfigLoader implements JestConfigLoader {
     private readonly projectRoot: string
   ) {}
 
-  public loadConfig(): Jest.Configuration {
+  public loadConfig(): Config.InitialOptions {
     const jestConfig = this.readConfigFromJestConfigFile() || this.readConfigFromPackageJson() || {};
     return jestConfig;
   }

packages/jest-runner/src/configLoaders/JestConfigLoader.ts

@@ -6,11 +6,13 @@
  * The loaderConfig method will return a usable config for Jest to use.
  */
 
+import { Config } from '@jest/types';
+
 export default interface JestConfigLoader {
   /*
    * Load the JSON representation of a Jest Configuration.
    *
    * @return {JestConfiguration} an object containing the Jest configuration.
    */
-  loadConfig(): Jest.Configuration;
+  loadConfig(): Config.InitialOptions;
 }

packages/jest-runner/src/configLoaders/ReactScriptsJestConfigLoader.ts

@@ -1,6 +1,7 @@
 import path from 'path';
 
 import { tokens } from '@stryker-mutator/api/plugin';
+import { Config } from '@jest/types';
 
 import { createReactJestConfig } from '../utils/createReactJestConfig';
 import { projectRootToken, resolveToken } from '../pluginTokens';
@@ -12,7 +13,7 @@ export default class ReactScriptsJestConfigLoader implements JestConfigLoader {
 
   constructor(private readonly resolve: RequireResolve, private readonly projectRoot: string) {}
 
-  public loadConfig(): Jest.Configuration {
+  public loadConfig(): Config.InitialOptions {
     try {
       // Get the location of react script, this is later used to generate the Jest configuration used for React projects.
       const reactScriptsLocation = path.join(this.resolve('react-scripts/package.json'), '..');
@@ -33,7 +34,7 @@ export default class ReactScriptsJestConfigLoader implements JestConfigLoader {
     return arg.code !== undefined;
   }
 
-  private createJestConfig(reactScriptsLocation: string): Jest.Configuration {
+  private createJestConfig(reactScriptsLocation: string): Config.InitialOptions {
     return createReactJestConfig((relativePath: string): string => path.join(reactScriptsLocation, relativePath), this.projectRoot, false);
   }
 }