feat(spectral): rework default spectral rules and expose as static "i…

…bm:oas" ruleset

src/spectral/rulesets/.defaultsForSpectral.yaml

@@ -1,24 +1,94 @@
-extends: [[spectral:oas, off]]
-formats: [oas2, oas3]
-functionsDir: ../functions
+extends: spectral:oas
 rules:
+
+  # Original list created from Spectral with:
+  # jq -r '.rules | to_entries | .[] | select(.value.recommended != false) | "  \(.key): off"' src/rulesets/oas/index.json
+
+  # Turn off -- duplicates no_success_response_codes
+  operation-2xx-response: off
+  # Turn off - duplicates non-configurable validation - form-data.js
+  oas2-operation-formData-consume-check: off
+  # Turn off - duplicates non-configurable validation - operation-ids.js
+  operation-operationId-unique: off
+  # Turn off - duplicates non-configurable validation - operations-shared.js
+  operation-parameters: off
+  # Enable with same severity as Spectral
+  operation-tag-defined: true
+  # Turn off - duplicates missing_path_parameter
+  path-params: off
+  # Turn off - exclude from ibm:oas
+  info-contact: off
+  # Turn off - exclude from ibm:oas
+  info-description: off
+  # Enable with same severity as Spectral
   no-eval-in-markdown: true
+  # Enable with same severity as Spectral
   no-script-tags-in-markdown: true
+  # Enable with same severity as Spectral
   openapi-tags: true
+  # Enable with same severity as Spectral
   operation-description: true
+  # Turn off - duplicates operation_id_case_convention, operation_id_naming_convention
+  operation-operationId: off
+  # Turn off - duplicates operation_id_case_convention
+  operation-operationId-valid-in-url: off
+  # Enable with same severity as Spectral
   operation-tags: true
-  operation-tag-defined: true
+  # Turn off - duplicates missing_path_parameter
+  path-declarations-must-exist: off
+  # Enable with same severity as Spectral
   path-keys-no-trailing-slash: true
+  # Turn off - duplicates non-configurable validation - paths.js
+  path-not-include-query: off
+  # Turn off - duplicates $ref_siblings (off by default)
+  no-$ref-siblings: off
+  # Enable with same severity as Spectral
   typed-enum: true
+  # Enable with same severity as Spectral
   oas2-api-host: true
+  # Enable with same severity as Spectral
   oas2-api-schemes: true
+  # Enable with same severity as Spectral
   oas2-host-trailing-slash: true
-  oas2-valid-example: true
+  # Turn off - dupicates non-configurable validation - security-ibm.js
+  oas2-operation-security-defined: off
+  # Turn off
+  oas2-valid-parameter-example: off
+  # Enable with same severity as Spectral
   oas2-valid-definition-example: true
+  # Turn off
+  oas2-valid-response-example: off
+  # Turn off
+  oas2-valid-response-schema-example: off
+  # Enable with same severity as Spectral
   oas2-anyOf: true
+  # Enable with same severity as Spectral
   oas2-oneOf: true
+  # Turn off
+  oas2-schema: off
+  # Turn off - duplicates non-configurable validation in base validator
+  oas2-unused-definition: off
+  # Enable with same severity as Spectral
   oas3-api-servers: true
+  # Enable with same severity as Spectral
   oas3-examples-value-or-externalValue: true
+  # Turn off - dupicates non-configurable validation - security-ibm.js
+  oas3-operation-security-defined: off
+  # Enable with same severity as Spectral
   oas3-server-trailing-slash: true
-  oas3-valid-example: true
+  # Turn off
+  oas3-valid-oas-parameter-example: off
+  # Turn off
+  oas3-valid-oas-header-example: off
+  # Turn off
+  oas3-valid-oas-content-example: off
+  # Turn off
+  oas3-valid-parameter-schema-example: off
+  # Turn off
+  oas3-valid-header-schema-example: off
+  # Enable with same severity as Spectral
   oas3-valid-schema-example: true
+  # Turn off
+  oas3-schema: off
+  # Turn off - duplicates non-configurable validation in base validator
+  oas3-unused-components-schema: off

src/spectral/utils/spectral-validator.js

@@ -1,7 +1,9 @@
 const MessageCarrier = require('../../plugins/utils/messageCarrier');
 const config = require('../../cli-validator/utils/processConfiguration');
+const { Spectral } = require('@stoplight/spectral');
 const { isOpenApiv2, isOpenApiv3 } = require('@stoplight/spectral');
 const { mergeRules } = require('@stoplight/spectral/dist/rulesets');
+const fs = require('fs');
 // default spectral ruleset file
 const defaultSpectralRulesetURI =
   __dirname + '/../rulesets/.defaultsForSpectral.yaml';
@@ -59,6 +61,13 @@ const setup = async function(spectral, configObject) {
     return Promise.reject(message);
   }
 
+  // Add IBM default ruleset to static assets to allow extends to reference it
+  const staticAssets = require('@stoplight/spectral/rulesets/assets/assets.json');
+  const content = fs.readFileSync(defaultSpectralRulesetURI, 'utf8');
+  staticAssets['ibm:oas'] = content;
+  Spectral.registerStaticAssets(staticAssets);
+
+  // Register formats
   spectral.registerFormat('oas2', isOpenApiv2);
   spectral.registerFormat('oas3', isOpenApiv3);
 
@@ -67,10 +76,8 @@ const setup = async function(spectral, configObject) {
     defaultSpectralRulesetURI
   );
 
-  // Combine user ruleset with the default ruleset
-  // The defined user ruleset will take precendence over the default ruleset
-  // Any rules specified in both will have the user defined rule severity override the default rule severity
-  await spectral.loadRuleset([defaultSpectralRulesetURI, spectralRulesetURI]);
+  // Load either the user-defined ruleset or our default ruleset
+  await spectral.loadRuleset(spectralRulesetURI);
 
   // Combine default/user ruleset with the validaterc spectral rules
   // The validaterc rules will take precendence in the case of duplicate rules

test/spectral/mockFiles/mockConfig/custom-rules.yaml

@@ -0,0 +1,11 @@
+extends: ibm:oas
+rules:
+  oas3-request-body-example:
+    description: All request bodies should have an example.
+    formats: [oas3]
+    given: '$.paths..requestBody..content.*'
+    severity: warn
+    then:
+      function: xor
+      functionOptions:
+        properties: [ example, examples ]

test/spectral/mockFiles/mockConfig/extends-default.yaml

@@ -0,0 +1,8 @@
+extends: ibm:oas
+rules:
+  # Turn off a rule that is on in the default ruleset
+  openapi-tags: off
+  # Turn on a rule that is off in the default ruleset
+  no-eval-in-markdown : error
+  # Change the severity of a rule in the default ruleset
+  oas3-valid-schema-example: warn

test/spectral/tests/spectral-config.test.js

@@ -0,0 +1,152 @@
+const path = require('path');
+const commandLineValidator = require('../../../src/cli-validator/runValidator');
+const config = require('../../../src/cli-validator/utils/processConfiguration');
+const { getCapturedText } = require('../../test-utils');
+
+describe('Spectral - test custom configuration', function() {
+  it('test Spectral info and hint rules', async function() {
+    // Set config to mock .spectral.yml file before running
+    const mockPath = path.join(
+      __dirname,
+      '../mockFiles/mockConfig/info-and-hint.yaml'
+    );
+    const mockConfig = jest
+      .spyOn(config, 'getSpectralRuleset')
+      .mockReturnValue(mockPath);
+
+    const consoleSpy = jest.spyOn(console, 'log').mockImplementation(() => {});
+    // set up mock user input
+    const program = {};
+    program.args = ['./test/spectral/mockFiles/oas3/enabled-rules.yml'];
+    program.default_mode = true;
+    program.json = true;
+
+    // Note: validator does not set exitcode for jsonOutput
+    await commandLineValidator(program);
+
+    // Ensure mockConfig was called and revert it to its original state
+    expect(mockConfig).toHaveBeenCalled();
+    mockConfig.mockRestore();
+
+    const capturedText = getCapturedText(consoleSpy.mock.calls);
+    const jsonOutput = JSON.parse(capturedText);
+
+    consoleSpy.mockRestore();
+
+    // Verify errors
+    expect(jsonOutput['errors']['spectral'].length).toBe(2);
+
+    // Verify warnings
+    expect(jsonOutput['warnings']['spectral'].length).toBe(10);
+
+    // Verify infos
+    expect(jsonOutput['infos']['spectral'].length).toBe(5);
+    expect(jsonOutput['infos']['spectral'][0]['message']).toEqual(
+      'Markdown descriptions should not contain `<script>` tags.'
+    );
+    expect(jsonOutput['infos']['spectral'][4]['message']).toEqual(
+      'Operation tags should be defined in global tags.'
+    );
+
+    // Verify hints
+    expect(jsonOutput['hints']['spectral'].length).toBe(2);
+    expect(jsonOutput['hints']['spectral'][0]['message']).toEqual(
+      'OpenAPI object should have non-empty `tags` array.'
+    );
+    expect(jsonOutput['hints']['spectral'][1]['message']).toEqual(
+      'Operation should have non-empty `tags` array.'
+    );
+  });
+
+  it('test Spectral custom config that extends ibm:oas', async function() {
+    // Set config to mock .spectral.yml file before running
+    const mockPath = path.join(
+      __dirname,
+      '../mockFiles/mockConfig/extends-default.yaml'
+    );
+    const mockConfig = jest
+      .spyOn(config, 'getSpectralRuleset')
+      .mockReturnValue(mockPath);
+
+    const consoleSpy = jest.spyOn(console, 'log').mockImplementation(() => {});
+    // set up mock user input
+    const program = {};
+    program.args = ['./test/spectral/mockFiles/oas3/enabled-rules.yml'];
+    program.default_mode = true;
+    program.json = true;
+
+    // Note: validator does not set exitcode for jsonOutput
+    await commandLineValidator(program);
+
+    // Ensure mockConfig was called and revert it to its original state
+    expect(mockConfig).toHaveBeenCalled();
+    mockConfig.mockRestore();
+
+    const capturedText = getCapturedText(consoleSpy.mock.calls);
+    const jsonOutput = JSON.parse(capturedText);
+
+    consoleSpy.mockRestore();
+
+    // Verify errors
+    expect(jsonOutput['errors']['spectral'].length).toBe(1);
+    expect(jsonOutput['errors']['spectral'][0]['message']).toEqual(
+      'Markdown descriptions should not contain `eval(`.'
+    );
+
+    // Verify warnings
+    expect(jsonOutput['warnings']['spectral'].length).toBe(17);
+    const warnings = jsonOutput['warnings']['spectral'].map(w => w['message']);
+    // This warning should be turned off
+    expect(warnings).not.toContain(
+      'OpenAPI object should have non-empty `tags` array.'
+    );
+    // This was redefined from error to warning
+    expect(warnings).toContain(
+      '`number_of_coins` property type should be integer'
+    );
+  });
+
+  it('test Spectral custom config that extends ibm:oas with custom rules', async function() {
+    // Set config to mock .spectral.yml file before running
+    const mockPath = path.join(
+      __dirname,
+      '../mockFiles/mockConfig/custom-rules.yaml'
+    );
+    const mockConfig = jest
+      .spyOn(config, 'getSpectralRuleset')
+      .mockReturnValue(mockPath);
+
+    const consoleSpy = jest.spyOn(console, 'log').mockImplementation(() => {});
+    // set up mock user input
+    const program = {};
+    program.args = ['./test/spectral/mockFiles/oas3/enabled-rules.yml'];
+    program.default_mode = true;
+    program.json = true;
+
+    // Note: validator does not set exitcode for jsonOutput
+    await commandLineValidator(program);
+
+    // Ensure mockConfig was called and revert it to its original state
+    expect(mockConfig).toHaveBeenCalled();
+    mockConfig.mockRestore();
+
+    const capturedText = getCapturedText(consoleSpy.mock.calls);
+    const jsonOutput = JSON.parse(capturedText);
+
+    consoleSpy.mockRestore();
+
+    // Verify errors
+    expect(jsonOutput['errors']['spectral'].length).toBe(2);
+
+    // Verify warnings
+    expect(jsonOutput['warnings']['spectral'].length).toBe(21);
+    const warnings = jsonOutput['warnings']['spectral'].map(w => w['message']);
+    // This is the new warning -- there should be four occurrences
+    const warning = 'All request bodies should have an example.';
+    const occurrences = warnings.reduce(
+      (a, v) => (v === warning ? a + 1 : a),
+      0
+    );
+    expect(occurrences).toBe(4);
+  });
+});