fix(exit prematurely): exit when no tests were executed (#2380)

BREAKING CHANGE: Stryker will now exit with exit code 1 when no tests were executed in the initial test run.

e2e/test/exit-prematurely-no-tests-executed/package.json

@@ -6,7 +6,7 @@
   "main": "index.js",
   "scripts": {
     "pretest": "rimraf \"reports\" stryker.log .stryker-tmp ",
-    "test": "stryker run",
+    "test": "stryker run || node -e \"require('fs').appendFileSync('stryker.log', 'Exit with non-zero exit code');\"",
     "posttest": "mocha --require ../../tasks/ts-node-register.js verify/*.ts"
   },
   "keywords": [],

e2e/test/exit-prematurely-no-tests-executed/src/add.js

@@ -0,0 +1,6 @@
+module.exports.add = function(num1, num2) {
+  return num1 + num2;
+};
+
+
+

e2e/test/exit-prematurely-no-tests-executed/stryker.conf.json

@@ -0,0 +1,12 @@
+{
+  "$schema": "../../node_modules/@stryker-mutator/core/schema/stryker-schema.json",
+  "testRunner": "mocha",
+  "concurrency": 2,
+  "coverageAnalysis": "perTest",
+  "reporters_comment": "Don't remove the html reporter here, we want to test that reports is not written",
+  "reporters": ["html"],
+  "plugins": [
+    "@stryker-mutator/mocha-runner"
+  ],
+  "fileLogLevel": "info"
+}

e2e/test/exit-prematurely-no-tests-executed/verify/verify.ts

@@ -5,12 +5,15 @@ describe('Verify stryker has ran correctly', () => {
 
   const strykerLog = fs.readFileSync('./stryker.log', 'utf8');
 
-  it('exit prematurely', async () => {
+  it('exit prematurely', () => {
     expect(strykerLog).contains('No tests were executed. Stryker will exit prematurely.');
   });
 
+  it('should exit with a non-zero exit code', () => {
+    expect(strykerLog).contains('Exit with non-zero exit code');
+  });
 
-  it('should warn about the globbing expression resulting in no files', () => {
-    expect(strykerLog).contains('Globbing expression "src/*.js" did not result in any files.');
+  it('should not report the mutant run', () => {
+    expect(fs.existsSync('reports'), 'Expected no reports to be written to disk, but they did').false;
   });
 });

e2e/test/grunt-stryker-test/stryker.conf.js

@@ -3,12 +3,14 @@ module.exports = function (config) {
     mutate: [
       'sampleProject/src/**'
     ],
-    karmaConfig: {
-      files: ['sampleProject/**']
+    karma: {
+      config: {
+        files: ['sampleProject/**']
+      }
     },
     testFramework: 'jasmine',
     testRunner: 'karma',
     logLevel: 'info',
     concurrency: 2
   });
-}
+}

packages/core/src/Stryker.ts

@@ -37,9 +37,11 @@ export default class Stryker {
       // 3. Perform a 'dry run' (initial test run). Runs the tests without active mutants and collects coverage.
       const dryRunExecutor = dryRunExecutorInjector.injectClass(DryRunExecutor);
       const mutationRunExecutorInjector = await dryRunExecutor.execute();
+
       // 4. Actual mutation testing. Will check every mutant and if valid run it in an available test runner.
       const mutationRunExecutor = mutationRunExecutorInjector.injectClass(MutationTestExecutor);
       const mutantResults = await mutationRunExecutor.execute();
+
       return mutantResults;
     } catch (error) {
       const log = loggerProvider.resolve(commonTokens.getLogger)(Stryker.name);

packages/core/src/process/3-DryRunExecutor.ts

@@ -70,7 +70,6 @@ export class DryRunExecutor {
     commonTokens.logger,
     commonTokens.options,
     coreTokens.timer,
-    coreTokens.mutants,
     coreTokens.concurrencyTokenProvider
   );
 
@@ -79,7 +78,6 @@ export class DryRunExecutor {
     private readonly log: Logger,
     private readonly options: StrykerOptions,
     private readonly timer: I<Timer>,
-    private readonly mutants: readonly Mutant[],
     private readonly concurrencyTokenProvider: I<ConcurrencyTokenProvider>
   ) {}
 
@@ -95,8 +93,8 @@ export class DryRunExecutor {
     this.validateResultCompleted(dryRunResult);
     const timing = this.calculateTiming(grossTimeMS, dryRunResult.tests);
     this.logInitialTestRunSucceeded(dryRunResult.tests, timing);
-    if (!dryRunResult.tests.length || !this.mutants.length) {
-      this.logTraceLogLevelHint();
+    if (!dryRunResult.tests.length) {
+      throw new ConfigError('No tests were executed. Stryker will exit prematurely. Please check your configuration.');
     }
     return testRunnerInjector
       .provideValue(coreTokens.timeOverheadMS, timing.overhead)
@@ -105,12 +103,6 @@ export class DryRunExecutor {
       .provideFactory(coreTokens.mutantsWithTestCoverage, findMutantTestCoverage);
   }
 
-  private logTraceLogLevelHint() {
-    if (!this.log.isTraceEnabled()) {
-      this.log.info('Trouble figuring out what went wrong? Try `npx stryker run --fileLogLevel trace --logLevel debug` to get some more info.');
-    }
-  }
-
   private validateResultCompleted(runResult: DryRunResult): asserts runResult is CompleteDryRunResult {
     switch (runResult.status) {
       case DryRunStatus.Complete:
@@ -119,9 +111,6 @@ export class DryRunExecutor {
           this.logFailedTestsInInitialRun(failedTests);
           throw new ConfigError('There were failed tests in the initial test run.');
         }
-        if (runResult.tests.length === 0) {
-          this.log.warn('No tests were executed. Stryker will exit prematurely. Please check your configuration.');
-        }
         return;
       case DryRunStatus.Error:
         this.logErrorsInInitialRun(runResult);

packages/core/test/unit/process/3-DryRunExecutor.spec.ts

@@ -2,13 +2,8 @@ import { EOL } from 'os';
 
 import { Injector } from 'typed-inject';
 import { factory, testInjector } from '@stryker-mutator/test-helpers';
-
-import { Mutant } from '@stryker-mutator/api/core';
-
 import sinon = require('sinon');
-
 import { TestRunner2, CompleteDryRunResult, ErrorDryRunResult, TimeoutDryRunResult } from '@stryker-mutator/api/test_runner2';
-
 import { expect } from 'chai';
 
 import Timer from '../../../src/utils/Timer';
@@ -23,7 +18,6 @@ describe(DryRunExecutor.name, () => {
   let testRunnerPoolMock: PoolMock<TestRunner2>;
   let sut: DryRunExecutor;
   let timerMock: sinon.SinonStubbedInstance<Timer>;
-  let mutants: Mutant[];
   let testRunnerMock: sinon.SinonStubbedInstance<Required<TestRunner2>>;
   let concurrencyTokenProviderMock: sinon.SinonStubbedInstance<ConcurrencyTokenProvider>;
 
@@ -33,10 +27,9 @@ describe(DryRunExecutor.name, () => {
     testRunnerPoolMock = createTestRunnerPoolMock();
     testRunnerPoolMock.worker$.next(testRunnerMock);
     concurrencyTokenProviderMock = sinon.createStubInstance(ConcurrencyTokenProvider);
-    mutants = [];
     injectorMock = factory.injector();
     injectorMock.resolve.withArgs(coreTokens.testRunnerPool).returns(testRunnerPoolMock);
-    sut = new DryRunExecutor(injectorMock, testInjector.logger, testInjector.options, timerMock, mutants, concurrencyTokenProviderMock);
+    sut = new DryRunExecutor(injectorMock, testInjector.logger, testInjector.options, timerMock, concurrencyTokenProviderMock);
   });
 
   it('should pass through any rejections', async () => {
@@ -54,6 +47,7 @@ describe(DryRunExecutor.name, () => {
     });
 
     it('should log about that this might take a while', async () => {
+      runResult.tests.push(factory.successTestResult());
       await sut.execute();
       expect(testInjector.logger.info).calledWith('Starting initial test run. This may take a while.');
     });
@@ -111,8 +105,8 @@ describe(DryRunExecutor.name, () => {
       });
 
       it('should log when there were no tests', async () => {
-        await sut.execute();
-        expect(testInjector.logger.warn).to.have.been.calledWith(
+        await expect(sut.execute()).rejectedWith(
+          ConfigError,
           'No tests were executed. Stryker will exit prematurely. Please check your configuration.'
         );
       });