feat(validation): add validation on plugin options (#2158)

A Stryker plugin is now able to contribute to the validation of StrykerOptions. Export a JSON schema from your plugin with the name `strykerValidationSchema` and it will be added to the validation. Also some housekeeping: 🏡 * StrykerOptions now support additional options as `[k: string]: unknown` instead of `[k: string: any`. This added about 100 compiler errors that I had to fix. * We apparently had some hidden features, like options for the clear text reporter and event recorder reporter. They are now documented in the core schema Example: ``` $ npm run stryker 13:17:48 (24856) INFO ConfigReader Using stryker.conf.js 13:17:49 (24856) ERROR OptionsValidator Config option "mochaOptions['async-only']" has the wrong type. It should be a boolean, but was a string. 13:17:49 (24856) ERROR OptionsValidator Config option "tsconfig" has the wrong type. It should be a object, but was a string. 13:17:49 (24856) ERROR StrykerCli Please correct these configuration errors and try again. ```
stryker-mutator · Apr 24, 2020 · d78fe1e · d78fe1e
1 parent be2d0cc
commit d78fe1e
Show file tree

Hide file tree

Showing 112 changed files with 955 additions and 428 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -1,4 +1,5 @@
 [{*.ts,*.js,*jsx,*tsx,*.json,*.code-workspace}]
 insert_final_newline = true
 indent_style = space
-indent_size = 2
+indent_size = 2
+end_of_line = lf
diff --git a/e2e/helpers.ts b/e2e/helpers.ts
@@ -19,6 +19,10 @@ type WritableMetricsResult = {
   -readonly [K in keyof MetricsResult]: MetricsResult[K];
 };
 
+export function readLogFile(fileName = path.resolve('stryker.log')): Promise<string> {
+  return fs.readFile(fileName, 'utf8');
+}
+
 export async function expectMetricsResult(expectedMetricsResult: Partial<MetricsResult>) {
   const actualMetricsResult = await readMutationTestResult();
   const actualSnippet: Partial<WritableMetricsResult> = {};

diff --git a/e2e/test/plugin-options-validation/package.json b/e2e/test/plugin-options-validation/package.json
@@ -0,0 +1,7 @@
+{
+  "scripts": {
+    "pretest": "rimraf stryker.log",
+    "test": "stryker run --fileLogLevel info stryker-error-in-plugin-options.conf.json || exit 0",
+    "posttest": "mocha --require \"ts-node/register\" verify/verify.ts"
+  }
+}
diff --git a/e2e/test/plugin-options-validation/stryker-error-in-plugin-options.conf.json b/e2e/test/plugin-options-validation/stryker-error-in-plugin-options.conf.json
@@ -0,0 +1,21 @@
+{
+  "$schema": "https://raw.githubusercontent.com/stryker-mutator/stryker/master/packages/api/schema/stryker-core.json",
+  "mochaOptions": {
+    "spec": "this should have been an array"
+  },
+  "tsconfigFile": 42,
+  "babel": {
+    "extensions": "should be an array"
+  },
+  "jasmineConfigFile": {
+    "should": "be a string"
+  },
+  "karma": {
+    "projectType": "Project type not supported"
+  },
+  "webpack": {
+    "configFile": [
+      "should be a string"
+    ]
+  }
+}
diff --git a/e2e/test/plugin-options-validation/verify/verify.ts b/e2e/test/plugin-options-validation/verify/verify.ts
@@ -0,0 +1,16 @@
+import { expect } from 'chai';
+import { readLogFile } from '../../../helpers';
+
+describe('Verify errors', () => {
+
+  it('should report the expected errors', async () => {
+    const logFile = await readLogFile();
+    expect(logFile).includes('Config option "mochaOptions.spec" has the wrong type');
+    expect(logFile).includes('Config option "tsconfigFile" has the wrong type');
+    expect(logFile).includes('Config option "babel.extensions" has the wrong type');
+    expect(logFile).includes('Config option "jasmineConfigFile" has the wrong type');
+    expect(logFile).not.includes('Config option "karma.projectType" has the wrong type');
+    expect(logFile).includes('Config option "karma.projectType" should be one of the allowed values');
+    expect(logFile).includes('Config option "webpack.configFile" has the wrong type');
+  });
+});
diff --git a/packages/api/schema/stryker-core.json b/packages/api/schema/stryker-core.json
@@ -38,6 +38,28 @@
         "MutationScore"
       ]
     },
+    "clearTextReporterOptions": {
+      "title": "ClearTextReporterOptions",
+      "type": "object",
+      "properties": {
+        "allowColor": {
+          "description": "Indicates whether or not to use color coding in output.",
+          "type": "boolean",
+          "default": true
+        },
+        "logTests": {
+          "description": "Indicates whether or not to log which tests were executed for a given mutant.",
+          "type": "boolean",
+          "default": true
+        },
+        "maxTestsToLog": {
+          "description": "Indicates the maximum amount of test to log when `logTests` is enabled",
+          "type": "integer",
+          "minimum": 0,
+          "default": 3
+        }
+      }
+    },
     "dashboardOptions": {
       "title": "DashboardOptions",
       "additionalProperties": false,
@@ -67,6 +89,18 @@
         }
       }
     },
+    "eventRecorderOptions": {
+      "title": "EventRecorderOptions",
+      "additionalProperties": false,
+      "type": "object",
+      "properties": {
+        "baseDir": {
+          "description": "The base dir to write the events to",
+          "type": "string",
+          "default": "reports/mutation/events"
+        }
+      }
+    },
     "htmlReporterOptions": {
       "title": "HtmlReporterOptions",
       "additionalProperties": false,
@@ -164,11 +198,21 @@
       ],
       "default": "off"
     },
+    "clearTextReporter": {
+      "description": "The options for the clear text reporter.",
+      "$ref": "#/definitions/clearTextReporterOptions",
+      "default": {}
+    },
     "dashboard": {
-      "description": "The options for the dashboard reporter",
+      "description": "The options for the dashboard reporter.",
       "$ref": "#/definitions/dashboardOptions",
       "default": {}
     },
+    "eventReporter": {
+      "description": "The options for the event recorder reporter.",
+      "$ref": "#/definitions/eventRecorderOptions",
+      "default": {}
+    },
     "fileLogLevel": {
       "description": "Set the log level that Stryker uses to write to the \"stryker.log\" file",
       "$ref": "#/definitions/logLevel",

diff --git a/packages/api/src/core/OptionsEditor.ts b/packages/api/src/core/OptionsEditor.ts
@@ -9,7 +9,7 @@ import { StrykerOptions } from '../../core';
  * editing of the configuration object is done by reference.
  *
  */
-export interface OptionsEditor {
+export interface OptionsEditor<T extends StrykerOptions = StrykerOptions> {
   /**
    * Extending classes only need to implement the edit method, this method
    * receives a writable config object that can be edited in any way.
@@ -18,5 +18,5 @@ export interface OptionsEditor {
    *
    * @param options: The stryker configuration object
    */
-  edit(options: StrykerOptions): void;
+  edit(options: T): void;
 }
diff --git a/packages/api/src/plugin/Plugins.ts b/packages/api/src/plugin/Plugins.ts
@@ -105,4 +105,5 @@ export type Plugins = {
 export interface PluginResolver {
   resolve<T extends keyof Plugins>(kind: T, name: string): Plugins[T];
   resolveAll<T extends keyof Plugins>(kind: T): Array<Plugins[T]>;
+  resolveValidationSchemaContributions(): object[];
 }
diff --git a/packages/api/testResources/module/useCore.ts b/packages/api/testResources/module/useCore.ts
@@ -18,6 +18,14 @@ const optionsAllArgs: StrykerOptions = {
   thresholds: { high: 80, low: 20, break: null},
   timeoutFactor: 1.5,
   timeoutMS: 5000,
+  clearTextReporter: {
+    allowColor: true,
+    logTests: true,
+    maxTestsToLog: 3,
+  },
+  eventReporter: {
+    baseDir: 'reports/mutation/events'
+  },
   transpilers: [],
   dashboard: {
     baseUrl: 'baseUrl',

diff --git a/packages/babel-transpiler/package.json b/packages/babel-transpiler/package.json
@@ -4,7 +4,7 @@
   "description": "A plugin for babel projects using Stryker",
   "main": "src/index.js",
   "scripts": {
-    "test": "nyc --exclude-after-remap=false --check-coverage --reporter=html --report-dir=reports/coverage --lines 90 --functions 90 --branches 80 npm run mocha",
+    "test": "nyc --exclude-after-remap=false --check-coverage --reporter=html --report-dir=reports/coverage --lines 90 --functions 80 --branches 80 npm run mocha",
     "mocha": "mocha \"test/helpers/**/*.js\" \"test/unit/**/*.js\" && mocha --timeout 100000 \"test/helpers/**/*.js\" \"test/integration/**/*.js\"",
     "stryker": "node ../core/bin/stryker run"
   },

diff --git a/packages/babel-transpiler/src/BabelConfigReader.ts b/packages/babel-transpiler/src/BabelConfigReader.ts
@@ -1,48 +1,30 @@
 import * as fs from 'fs';
 import * as path from 'path';
 
-import { StrykerOptions } from '@stryker-mutator/api/core';
 import { Logger } from '@stryker-mutator/api/logging';
 import { commonTokens, tokens } from '@stryker-mutator/api/plugin';
 
-import * as babel from './helpers/babelWrapper';
-
-export interface StrykerBabelConfig {
-  extensions: readonly string[];
-  options: babel.TransformOptions;
-  optionsFile: string | null;
-  optionsApi?: Partial<babel.ConfigAPI>;
-}
-
-export const CONFIG_KEY = 'babel';
-export const FILE_KEY: keyof StrykerBabelConfig = 'optionsFile';
-export const OPTIONS_KEY: keyof StrykerBabelConfig = 'options';
-export const EXTENSIONS_KEY: keyof StrykerBabelConfig = 'extensions';
+import { StrykerBabelConfig } from '../src-generated/babel-transpiler-options';
 
-const DEFAULT_BABEL_CONFIG: Readonly<StrykerBabelConfig> = Object.freeze({
-  extensions: Object.freeze([]),
-  options: Object.freeze({}),
-  optionsFile: '.babelrc',
-});
+import * as babel from './helpers/babelWrapper';
+import { BabelTranspilerWithStrykerOptions } from './BabelTranspilerWithStrykerOptions';
+import { ConfigAPI } from './helpers/babelWrapper';
 
 export class BabelConfigReader {
   public static inject = tokens(commonTokens.logger);
   constructor(private readonly log: Logger) {}
 
-  public readConfig(strykerOptions: StrykerOptions): StrykerBabelConfig {
-    const babelConfig: StrykerBabelConfig = {
-      ...DEFAULT_BABEL_CONFIG,
-      ...strykerOptions[CONFIG_KEY],
-    };
+  public readConfig(strykerOptions: BabelTranspilerWithStrykerOptions): StrykerBabelConfig {
+    const babelConfig = { ...strykerOptions.babel };
     babelConfig.options = {
-      ...this.readBabelOptionsFromFile(babelConfig.optionsFile, babelConfig.optionsApi),
+      ...this.readBabelOptionsFromFile(babelConfig.optionsFile),
       ...babelConfig.options,
     };
     this.log.debug(`Babel config is: ${JSON.stringify(babelConfig, null, 2)}`);
     return babelConfig;
   }
 
-  private readBabelOptionsFromFile(relativeFileName: string | null, optionsApi?: Partial<babel.ConfigAPI>): babel.TransformOptions {
+  private readBabelOptionsFromFile(relativeFileName: string | null): babel.TransformOptions {
     if (relativeFileName) {
       const babelrcPath = path.resolve(relativeFileName);
       this.log.debug(`Reading .babelrc file from path "${babelrcPath}"`);
@@ -55,7 +37,7 @@ export class BabelConfigReader {
             const config = require(babelrcPath);
             if (typeof config === 'function') {
               const configFunction = config as babel.ConfigFunction;
-              return configFunction(optionsApi as babel.ConfigAPI);
+              return configFunction(noopBabelConfigApi);
             } else {
               return config as babel.TransformOptions;
             }
@@ -71,3 +53,24 @@ export class BabelConfigReader {
     return {};
   }
 }
+
+function noop() {}
+
+const noopBabelConfigApi: ConfigAPI = {
+  assertVersion() {
+    return true;
+  },
+  cache: {
+    forever: noop,
+    invalidate() {
+      return noop as any;
+    },
+    never: noop,
+    using() {
+      return noop as any;
+    },
+  },
+  env: noop as any,
+  caller: noop as any,
+  version: noop as any,
+};
diff --git a/packages/babel-transpiler/src/BabelTranspiler.ts b/packages/babel-transpiler/src/BabelTranspiler.ts
@@ -5,9 +5,12 @@ import { commonTokens, Injector, tokens, TranspilerPluginContext } from '@stryke
 import { Transpiler } from '@stryker-mutator/api/transpile';
 import { StrykerError } from '@stryker-mutator/util';
 
-import { BabelConfigReader, StrykerBabelConfig } from './BabelConfigReader';
+import { StrykerBabelConfig } from '../src-generated/babel-transpiler-options';
+
+import { BabelConfigReader } from './BabelConfigReader';
 import * as babel from './helpers/babelWrapper';
 import { toJSFileName } from './helpers/helpers';
+import { BabelTranspilerWithStrykerOptions } from './BabelTranspilerWithStrykerOptions';
 
 const DEFAULT_EXTENSIONS: readonly string[] = babel.DEFAULT_EXTENSIONS;
 
@@ -28,7 +31,7 @@ export class BabelTranspiler implements Transpiler {
         `Invalid \`coverageAnalysis\` "${options.coverageAnalysis}" is not supported by the stryker-babel-transpiler. Not able to produce source maps yet. Please set it to "off".`
       );
     }
-    this.babelConfig = babelConfigReader.readConfig(options);
+    this.babelConfig = babelConfigReader.readConfig(options as BabelTranspilerWithStrykerOptions);
     this.projectRoot = this.determineProjectRoot();
     this.extensions = [...DEFAULT_EXTENSIONS, ...this.babelConfig.extensions];
   }

diff --git a/packages/babel-transpiler/src/BabelTranspilerWithStrykerOptions.ts b/packages/babel-transpiler/src/BabelTranspilerWithStrykerOptions.ts
@@ -0,0 +1,5 @@
+import { StrykerOptions } from '@stryker-mutator/api/core';
+
+import { BabelTranspilerOptions } from '../src-generated/babel-transpiler-options';
+
+export interface BabelTranspilerWithStrykerOptions extends BabelTranspilerOptions, StrykerOptions {}
diff --git a/packages/babel-transpiler/src/index.ts b/packages/babel-transpiler/src/index.ts
@@ -2,4 +2,6 @@ import { declareFactoryPlugin, PluginKind } from '@stryker-mutator/api/plugin';
 
 import { babelTranspilerFactory } from './BabelTranspiler';
 
+export * as strykerValidationSchema from '../schema/babel-transpiler-options.json';
+
 export const strykerPlugins = [declareFactoryPlugin(PluginKind.Transpiler, 'babel', babelTranspilerFactory)];
diff --git a/packages/babel-transpiler/test/helpers/factories.ts b/packages/babel-transpiler/test/helpers/factories.ts
@@ -0,0 +1,10 @@
+import { StrykerBabelConfig } from '../../src-generated/babel-transpiler-options';
+
+export function createStrykerBabelConfig(overrides?: Partial<StrykerBabelConfig>): StrykerBabelConfig {
+  return {
+    extensions: ['.js', '.jsx', '.es6', '.es', '.mjs'],
+    optionsFile: '.babelrc',
+    options: {},
+    ...overrides,
+  };
+}
diff --git a/packages/babel-transpiler/test/integration/BabelProjects.it.spec.ts b/packages/babel-transpiler/test/integration/BabelProjects.it.spec.ts
@@ -1,14 +1,15 @@
 import * as path from 'path';
 
-import { ConfigAPI } from '@babel/core';
 import { File } from '@stryker-mutator/api/core';
 import { commonTokens } from '@stryker-mutator/api/plugin';
 import { testInjector } from '@stryker-mutator/test-helpers';
 import { expect } from 'chai';
 
-import { CONFIG_KEY, StrykerBabelConfig } from '../../src/BabelConfigReader';
 import { BabelTranspiler, babelTranspilerFactory } from '../../src/BabelTranspiler';
 import { ProjectLoader } from '../helpers/projectLoader';
+import { StrykerBabelConfig } from '../../src-generated/babel-transpiler-options';
+import { BabelTranspilerWithStrykerOptions } from '../../src/BabelTranspilerWithStrykerOptions';
+import { createStrykerBabelConfig } from '../helpers/factories';
 
 function describeIntegrationTest(projectName: string, babelConfig: Partial<StrykerBabelConfig> = {}) {
   const projectDir = path.resolve(__dirname, '..', '..', 'testResources', projectName);
@@ -20,7 +21,7 @@ function describeIntegrationTest(projectName: string, babelConfig: Partial<Stryk
   beforeEach(async () => {
     projectFiles = await ProjectLoader.getFiles(path.join(projectDir, 'source'));
     resultFiles = await ProjectLoader.getFiles(path.join(projectDir, 'expectedResult'));
-    testInjector.options[CONFIG_KEY] = babelConfig;
+    ((testInjector.options as unknown) as BabelTranspilerWithStrykerOptions).babel = createStrykerBabelConfig(babelConfig);
     babelTranspiler = testInjector.injector.provideValue(commonTokens.produceSourceMaps, false).injectFunction(babelTranspilerFactory);
   });
 
@@ -66,10 +67,8 @@ describe('Different extensions', () => {
   describeIntegrationTest('differentExtensions');
 });
 describe('A Babel project with babel.config.js config file that exports function', () => {
-  const noop = () => {};
   describeIntegrationTest('babelProjectWithBabelConfigJs', {
     extensions: ['.ts'],
-    optionsApi: { cache: { forever: noop } } as ConfigAPI,
     optionsFile: 'babel.config.js',
   });
 });