feat(api): rename test_runner2 -> test_runner (#2442)

BREAKING CHANGE: Plugin creators should now use `'test_runner'` instead of `'test_runner2'`.

packages/api/src/plugin/PluginKind.ts

@@ -3,7 +3,6 @@
  */
 export enum PluginKind {
   Checker = 'Checker',
-  TestRunner = 'TestRunner',
   TestRunner2 = 'TestRunner2',
   Reporter = 'Reporter',
 }

packages/api/src/plugin/Plugins.ts

@@ -1,8 +1,7 @@
 import { InjectableClass, InjectableFunction, InjectionToken } from 'typed-inject';
 
 import { Reporter } from '../../report';
-import { TestRunner } from '../../test_runner';
-import { TestRunner2 } from '../../test_runner2';
+import { TestRunner2 } from '../../test_runner';
 import { Checker } from '../../check';
 
 import { PluginContext } from './Contexts';
@@ -81,7 +80,6 @@ export function declareFactoryPlugin<TPluginKind extends PluginKind, Tokens exte
  */
 export interface PluginInterfaces {
   [PluginKind.Reporter]: Reporter;
-  [PluginKind.TestRunner]: TestRunner;
   [PluginKind.TestRunner2]: TestRunner2;
   [PluginKind.Checker]: Checker;
 }

packages/api/src/test_runner/RunOptions.ts

@@ -1,19 +1,23 @@
-/**
- * Represents an options object for a single run of a TestRunner.
- */
-interface RunOptions {
+import { Mutant, CoverageAnalysis } from '../../core';
+
+export interface RunOptions {
   /**
    * The amount of time (in milliseconds) the TestRunner has to complete the test run before a timeout occurs.
    */
   timeout: number;
+}
 
+export interface DryRunOptions extends RunOptions {
   /**
-   * The hooks JavaScript code that has to be loaded in the testing environment.
-   * It should be loaded right after the test framework but right before any tests can run.
+   * Indicates whether or not mutant coverage should be collected.
    */
-  testHooks?: string;
+  coverageAnalysis: CoverageAnalysis;
+}
 
-  mutatedFileName?: string;
+export interface MutantRunOptions extends RunOptions {
+  testFilter?: string[];
+  activeMutant: Mutant;
+  sandboxFileName: string;
 }
 
 export default RunOptions;

packages/api/src/test_runner/TestResult.ts

@@ -1,25 +1,34 @@
-import TestStatus from './TestStatus';
+import { TestStatus } from './TestStatus';
 
 /**
  * Indicates the result of a single test
  */
-interface TestResult {
+interface BaseTestResult {
   /**
-   * The full human readable name of the test
+   * The id of this test. Can be the name if the test runner doesn't have an 'id'
    */
-  name: string;
+  id: string;
   /**
-   * The status of the test
+   * The full human readable name of the test
    */
-  status: TestStatus;
+  name: string;
   /**
    * The time it took to run the test
    */
   timeSpentMs: number;
-  /**
-   * Optional: messages in case of status: Failed
-   */
-  failureMessages?: string[];
 }
 
-export default TestResult;
+export interface FailedTestResult extends BaseTestResult {
+  status: TestStatus.Failed;
+  failureMessage: string;
+}
+
+export interface SkippedTestResult extends BaseTestResult {
+  status: TestStatus.Skipped;
+}
+
+export interface SuccessTestResult extends BaseTestResult {
+  status: TestStatus.Success;
+}
+
+export type TestResult = SuccessTestResult | FailedTestResult | SkippedTestResult;

packages/api/src/test_runner/TestRunner.ts

@@ -1,58 +1,10 @@
-import RunOptions from './RunOptions';
-import RunResult from './RunResult';
-
-/**
- * Represents a TestRunner which can execute tests, resulting in a RunResult.
- *
- * A test runner should:
- *  - Report a `testResult` per test. See `TestResult` interface to know what is expected.
- *  - Report on code coverage after the initial test run (maybe, see below).
- *
- * ## A note on code coverage:
- *
- * TL;DR
- * If the test runner ran the tests in the same process as the test runner is spawned in, it doesn't have to do anything.
- * If it runs in a different process (i.e. a worker process) or uses a browser (i.e. karma) it needs to do something, see below.
- *
- * Collecting code coverage can improve the performance of a mutation test run quite a bit. Instead of running all tests for all mutants,
- * it can select which tests to run per mutant. Code coverage is a shared responsibility between Stryker and the test runner depending on the
- * `coverageStrategy`:
- *
- * 1. If `coverageAnalysis: 'off'`: No code coverage should be collected.
- * 2. If `coverageStrategy: 'all'`: Coverage should be collected for the entire test run.
- * 3. If `coverageStrategy: 'perTest'`: Coverage should be collected per test.
- *
- * For 2 and 3, Stryker will instrument the code with the istanbul code coverage engine during initial run.
- * In case of 3, Stryker will also inject beforeEach and afterEach hooks (specific to each test framework) in distinct between specific tests.
- *
- * At the end of the test run, the code coverage report is ready in a global variable called `__coverage__`. Node based test runners
- * which run their tests in the same process as the test runner is spawned in actually don't have to do any work, Stryker will be able
- * to pick up the report globally. However, if running in worker processes or a browser, it is the test runner's responsibility to
- * report the `__coverage__` at the end of the test run.
- *
- * If it doesn't exists globally, you don't have to do anything. In that case it's not an initial test run and there was no code instrumented.
- */
-interface TestRunner {
-  /**
-   * Optional. When implemented, will be called before runs are done on this test runner.
-   * @returns A promise if stuff is initialized asynchronously, runs will not start until the promise is resolved.
-   * Otherwise void
-   */
-  init?(): Promise<void> | void;
-
-  /**
-   * Executes a test run.
-   * @param options The options for this test run.
-   * @returns A promise to eventually complete the test run and deliver a RunResult.
-   */
-  run(options: RunOptions): Promise<RunResult>;
-
-  /**
-   * Optional. When implemented, will be called before the test runner's process is killed.
-   * @returns A promise if stuff is destroyed asynchronously, the runners process will not end until the promise is resolved.
-   * Otherwise void
-   */
-  dispose?(): Promise<void> | void;
+import { DryRunOptions, MutantRunOptions } from './RunOptions';
+import { DryRunResult } from './DryRunResult';
+import { MutantRunResult } from './MutantRunResult';
+
+export interface TestRunner2 {
+  init?(): Promise<void>;
+  dryRun(options: DryRunOptions): Promise<DryRunResult>;
+  mutantRun(options: MutantRunOptions): Promise<MutantRunResult>;
+  dispose?(): Promise<void>;
 }
-
-export default TestRunner;

packages/api/src/test_runner/TestStatus.ts

@@ -1,7 +1,7 @@
 /**
  * Indicates what the result of a single test was.
  */
-enum TestStatus {
+export enum TestStatus {
   /**
    * The test succeeded
    */
@@ -15,5 +15,3 @@ enum TestStatus {
    */
   Skipped,
 }
-
-export default TestStatus;

packages/api/test/unit/test_runner2/runResultHelpers.spec.ts

@@ -1,7 +1,7 @@
 import { expect } from 'chai';
 
-import { toMutantRunResult, DryRunStatus, MutantRunResult, MutantRunStatus } from '../../../test_runner2';
-import TestStatus from '../../../src/test_runner/TestStatus';
+import { toMutantRunResult, DryRunStatus, MutantRunResult, MutantRunStatus } from '../../../test_runner';
+import { TestStatus } from '../../../src/test_runner/TestStatus';
 
 describe('runResultHelpers', () => {
   describe(toMutantRunResult.name, () => {