feat: allow spectral rule severity to be modified in the validaterc (#…

…202)

src/.defaultsForValidator.js

@@ -106,6 +106,30 @@ const defaults = {
     'schemas': {
       'json_or_param_binary_string': 'warning'
     }
+  },
+  'spectral': {
+    'rules': {
+      'no-eval-in-markdown': "warning",
+      'no-script-tags-in-markdown': "warning",
+      'openapi-tags': "warning",
+      'operation-description': "warning",
+      'operation-tags': "warning",
+      'operation-tag-defined': "warning",
+      'path-keys-no-trailing-slash': "warning",
+      'typed-enum': "warning",
+      'oas2-api-host': "warning",
+      'oas2-api-schemes': "warning",
+      'oas2-host-trailing-slash': "warning",
+      'oas2-valid-example': "warning",
+      'oas2-valid-definition-example': "error",
+      'oas2-anyOf': "warning",
+      'oas2-oneOf': "warning",
+      'oas3-api-servers': "warning",
+      'oas3-examples-value-or-externalValue': "warning",
+      'oas3-server-trailing-slash': "warning",
+      'oas3-valid-example': "error",
+      'oas3-valid-schema-example': "error"
+    }
   }
 };
 

src/cli-validator/runValidator.js

@@ -166,7 +166,7 @@ const processInput = async function(program) {
   // or the default ruleset
   const spectral = new Spectral();
   try {
-    await spectralValidator.setup(spectral);
+    await spectralValidator.setup(spectral, configObject);
   } catch (err) {
     return Promise.reject(err);
   }

src/cli-validator/utils/processConfiguration.js

@@ -10,6 +10,8 @@ const defaultConfig = require('../../.defaultsForValidator');
 // global objects
 const readFile = util.promisify(fs.readFile);
 const defaultObject = defaultConfig.defaults;
+// Clear spectral rules from defaultObject, only use the explicit values in the validaterc file
+defaultObject.spectral.rules = {};
 const deprecatedRuleObject = defaultConfig.deprecated;
 const configOptions = defaultConfig.options;
 
@@ -42,6 +44,10 @@ const validateConfigObject = function(configObject, chalk) {
   const allowedSpecs = Object.keys(defaultObject);
   const userSpecs = Object.keys(configObject);
   userSpecs.forEach(spec => {
+    // Do not check "spectral" spec rules
+    if (spec === 'spectral') {
+      return;
+    }
     if (!allowedSpecs.includes(spec)) {
       validObject = false;
       configErrors.push({

src/lib/index.js

@@ -15,8 +15,8 @@ module.exports = async function(input, defaultMode = false) {
   const spectral = new Spectral();
 
   try {
-    await spectralValidator.setup(spectral);
     configObject = await config.get(defaultMode, chalk);
+    await spectralValidator.setup(spectral, configObject);
   } catch (err) {
     return Promise.reject(err);
   }

src/spectral/rulesets/.defaultsForSpectral.yaml

@@ -1,6 +1,6 @@
 extends: [[spectral:oas, off]]
 formats: [oas2, oas3]
-functionsDir": ../functions
+functionsDir: ../functions
 rules:
   no-eval-in-markdown: true
   no-script-tags-in-markdown: true

src/spectral/utils/spectral-validator.js

@@ -1,8 +1,9 @@
 const MessageCarrier = require('../../plugins/utils/messageCarrier');
 const config = require('../../cli-validator/utils/processConfiguration');
 const { isOpenApiv2, isOpenApiv3 } = require('@stoplight/spectral');
+const { mergeRules } = require('@stoplight/spectral/dist/rulesets');
 // default spectral ruleset file
-const defaultSpectralRuleset =
+const defaultSpectralRulesetURI =
   __dirname + '/../rulesets/.defaultsForSpectral.yaml';
 
 const parseResults = function(results, debug) {
@@ -46,7 +47,7 @@ const parseResults = function(results, debug) {
 };
 
 // setup: registers the oas2/oas3 formats, and attempts to load the ruleset file
-const setup = async function(spectral) {
+const setup = async function(spectral, configObject) {
   if (!spectral) {
     const message =
       'Error (spectral-validator): An instance of spectral has not been initialized.';
@@ -57,12 +58,22 @@ const setup = async function(spectral) {
   spectral.registerFormat('oas3', isOpenApiv3);
 
   // load the spectral ruleset, either a user's or the default ruleset
-  const spectralRuleset = await config.getSpectralRuleset(
-    defaultSpectralRuleset
+  const spectralRulesetURI = await config.getSpectralRuleset(
+    defaultSpectralRulesetURI
   );
 
+  // Combine user ruleset with the default ruleset
+  // The defined user ruleset will take precendence over the default ruleset
+  // Any rules specified in both will have the user defined rule severity override the default rule severity
+  await spectral.loadRuleset([defaultSpectralRulesetURI, spectralRulesetURI]);
+
+  // Combine default/user ruleset with the validaterc spectral rules
+  // The validaterc rules will take precendence in the case of duplicate rules
+  const userRules = Object.assign({}, spectral.rules); // Clone rules
   try {
-    return await spectral.loadRuleset(spectralRuleset);
+    return await spectral.setRules(
+      mergeRules(userRules, configObject.spectral.rules)
+    );
   } catch (err) {
     return Promise.reject(err);
   }

test/spectral/mockFiles/mockConfig/.mockSpectral.yaml

@@ -0,0 +1,24 @@
+extends: [[spectral:oas, off]]
+formats: [oas2, oas3]
+functionsDir: ../functions
+rules:
+  no-eval-in-markdown: error
+  no-script-tags-in-markdown: true
+  openapi-tags: true
+  operation-description: true
+  operation-tags: true
+  operation-tag-defined: true
+  path-keys-no-trailing-slash: true
+  typed-enum: true
+  oas2-api-host: true
+  oas2-api-schemes: true
+  oas2-host-trailing-slash: true
+  oas2-valid-example: true
+  oas2-valid-definition-example: true
+  oas2-anyOf: true
+  oas2-oneOf: true
+  oas3-api-servers: true
+  oas3-examples-value-or-externalValue: true
+  oas3-server-trailing-slash: true
+  oas3-valid-example: true
+  oas3-valid-schema-example: true

test/spectral/tests/spectral-validator.test.js

@@ -1,5 +1,11 @@
 const { getCapturedText } = require('../../test-utils');
 const spectralValidator = require('../../../src/spectral/utils/spectral-validator');
+const inCodeValidator = require('../../../src/lib');
+const oas3InMemory = require('../mockFiles/oas3/enabled-rules-in-memory');
+const path = require('path');
+const config = require('../../../src/cli-validator/utils/processConfiguration');
+const defaultConfig = require('../../../src/.defaultsForValidator');
+const defaultObject = defaultConfig.defaults;
 
 describe('spectral - test spectral-validator.js', function() {
   let consoleSpy;
@@ -108,3 +114,157 @@ describe('spectral - test spectral-validator.js', function() {
     );
   });
 });
+
+describe('spectral - test config file changes with .spectral.yml', function() {
+  let validationResults;
+  let errors;
+  let warnings;
+
+  beforeAll(async () => {
+    // Set config to mock .spectral.yml file before running
+    const mockPath = path.join(
+      __dirname,
+      '../mockFiles/mockConfig/.mockSpectral.yaml'
+    );
+    const mockConfig = jest
+      .spyOn(config, 'getSpectralRuleset')
+      .mockReturnValue(mockPath);
+
+    // Below is used from enabled-rules.test.js
+    // set up mock user input
+    const defaultMode = false;
+    validationResults = await inCodeValidator(oas3InMemory, defaultMode);
+
+    // Ensure mockConfig was called and revert it to its original state
+    expect(mockConfig).toHaveBeenCalled();
+    mockConfig.mockRestore();
+
+    // should produce an object with `errors` and `warnings` keys that should
+    // both be non-empty
+    expect(validationResults.errors.length).toBeGreaterThan(0);
+    expect(validationResults.warnings.length).toBeGreaterThan(0);
+
+    errors = validationResults.errors.map(error => error.message);
+    warnings = validationResults.warnings.map(warn => warn.message);
+    expect(errors.length).toBeGreaterThan(0);
+    expect(warnings.length).toBeGreaterThan(0);
+  });
+
+  // One test should be an error instead of a warning compared to the enable-rules.test.js
+  it('test no-eval-in-markdown rule using mockFiles/oas3/enabled-rules-in-memory', function() {
+    expect(errors).toContain(
+      'Markdown descriptions should not contain `eval(`.'
+    );
+    expect(warnings).not.toContain(
+      'Markdown descriptions should not contain `eval(`.'
+    );
+  });
+
+  // Other tests should be their default severity levels
+  it('test no-script-tags-in-markdown rule using mockFiles/oas3/enabled-rules-in-memory', function() {
+    expect(errors).not.toContain(
+      'Markdown descriptions should not contain `<script>` tags.'
+    );
+    expect(warnings).toContain(
+      'Markdown descriptions should not contain `<script>` tags.'
+    );
+  });
+});
+
+describe('spectral - test config file changes with .validaterc', function() {
+  let validationResults;
+  let errors;
+  let warnings;
+
+  beforeAll(async () => {
+    // Create a mockObject from the defaultObject and mock config.get()
+    const mockObject = Object.assign({}, defaultObject);
+    mockObject.spectral.rules['no-script-tags-in-markdown'] = 'error';
+    const mockConfig = jest.spyOn(config, 'get').mockReturnValue(mockObject);
+
+    // Below is used from enabled-rules.test.js
+    // set up mock user input
+    const defaultMode = false;
+    validationResults = await inCodeValidator(oas3InMemory, defaultMode);
+
+    // Ensure mockConfig was called and revert it to its original state
+    expect(mockConfig).toHaveBeenCalled();
+    mockConfig.mockRestore();
+
+    // should produce an object with `errors` and `warnings` keys that should
+    // both be non-empty
+    expect(validationResults.errors.length).toBeGreaterThan(0);
+    expect(validationResults.warnings.length).toBeGreaterThan(0);
+
+    errors = validationResults.errors.map(error => error.message);
+    warnings = validationResults.warnings.map(warn => warn.message);
+    expect(errors.length).toBeGreaterThan(0);
+    expect(warnings.length).toBeGreaterThan(0);
+  });
+
+  // One test should be an error instead of a warning compared to the enable-rules.test.js
+  it('test no-script-tags-in-markdown rule using mockFiles/oas3/enabled-rules-in-memory', function() {
+    expect(errors).toContain(
+      'Markdown descriptions should not contain `<script>` tags.'
+    );
+    expect(warnings).not.toContain(
+      'Markdown descriptions should not contain `<script>` tags.'
+    );
+  });
+
+  // Other tests should be their default severity levels
+  it('test no-eval-in-markdown rule using mockFiles/oas3/enabled-rules-in-memory', function() {
+    expect(errors).not.toContain(
+      'Markdown descriptions should not contain `eval(`.'
+    );
+    expect(warnings).toContain(
+      'Markdown descriptions should not contain `eval(`.'
+    );
+  });
+});
+
+describe('spectral - test config file changes with .validaterc, all rules off', function() {
+  let validationResults;
+
+  beforeAll(async () => {
+    // Create a mockObject from the defaultObject and mock config.get()
+    const mockObject = Object.assign({}, defaultObject);
+    mockObject.spectral.rules = {
+      'no-eval-in-markdown': 'off',
+      'no-script-tags-in-markdown': 'off',
+      'openapi-tags': 'off',
+      'operation-description': 'off',
+      'operation-tags': 'off',
+      'operation-tag-defined': 'off',
+      'path-keys-no-trailing-slash': 'off',
+      'typed-enum': 'off',
+      'oas2-api-host': 'off',
+      'oas2-api-schemes': 'off',
+      'oas2-host-trailing-slash': 'off',
+      'oas2-valid-example': 'off',
+      'oas2-valid-definition-example': 'off',
+      'oas2-anyOf': 'off',
+      'oas2-oneOf': 'off',
+      'oas3-api-servers': 'off',
+      'oas3-examples-value-or-externalValue': 'off',
+      'oas3-server-trailing-slash': 'off',
+      'oas3-valid-example': 'off',
+      'oas3-valid-schema-example': 'off'
+    };
+    const mockConfig = jest.spyOn(config, 'get').mockReturnValue(mockObject);
+
+    // set up mock user input
+    const defaultMode = false;
+    validationResults = await inCodeValidator(oas3InMemory, defaultMode);
+
+    // Ensure mockConfig was called and revert it to its original state
+    expect(mockConfig).toHaveBeenCalled();
+    mockConfig.mockRestore();
+  });
+
+  // There should be no errors and 2 warnings for a non-spectral rule
+  it('test no spectral errors and no spectral warnings', function() {
+    expect(validationResults.errors.length).toBe(0);
+    expect(validationResults.warnings.length).toBe(2);
+  });
+});