spectral-test-suite-v0.1.json-schema.yaml

$schema: "https://json-schema.org/draft/2020-12/schema"
title: Spectral Ruleset Test Suite
description: |
  A machine-readable description of tests to run to ensure a Spectral Ruleset or a subset
  or it works as intended. In its current version, the format only deals with
  rules, not aliases or extends.
  
  A single test suite file may or may not contain all of the tests for all rules contained
  in a ruleset. For maintainability and readability, it could be interesting that a test 
  suite focuses on a single rule, and so have as many test suite files as rules.
  Though it would be possible to split the tests of a single rule across test suite, it is
  not recommended to do so. 
type: object
unevaluatedProperties: false
required:
  - spectralTestSuite
  - ruleset
  - rules
properties:
  spectralTestSuite:
    description: The version of Spectral Ruleset Test Suite used
    allOf:
      - $ref: "#/$defs/VersionNumber"
      - enum:
        - "0.1"
  ruleset:
    description: |
      The relative or absolute filepath or url to the Spectral Ruleset file containing
      the elements to be tested. The file can be either in YAML or JSON format.
    type: string
  rules:
    description: |
      Rule tests. Each key of this object corresponds to a rule name,
      hence the the keys of rules in a Spectral ruleset.
    type: object
    additionalProperties:
      $ref: "#/$defs/RuleTests"
$defs:
  VersionNumber:
    type: string
  OpenApiVersions:
    type: array
    default:
      - "2.0"
      - "3.0"
      - "3.1"
    items:
      allOf:
        - $ref: "#/$defs/VersionNumber"
        - enum:
          - "2.0"
          - "3.0"
          - "3.1"
  AsyncApiVersions:
    type: array
    default: [ "2.0", "2.1", "2.2", "2.3", "2.4", "2.5", "2.6" ]
    items:
      allOf:
        - $ref: "#/$defs/VersionNumber"
        - enum: [ "2.0", "2.1", "2.2", "2.3", "2.4", "2.5", "2.6" ]
  JsonSchemaVersions:
    type: array
    default: [ 2020-12 ] 
    items:
      allOf:
        - $ref: "#/$defs/VersionNumber"
        - enum: 
          - $schema
          - loose
          - draft4
          - draft6
          - draft7
          - 2019-09
          - 2020-12
  RuleTests:
    description: Tests to validate a Spectral rule works as intended 
    type: object
    unevaluatedProperties: false
    allOf:
      - if:
          properties:
            format:
              const: 'openapi'
        then:
          properties:
            versions:
              $ref: "#/$defs/OpenApiVersions"
      - if:
          properties:
            format:
              const: 'asyncapi'
        then:
          properties:
            versions:
              $ref: "#/$defs/AsyncApiVersions"
      - if:
          properties:
            format:
              const: 'jsonschema'
        then:
          properties:
            versions:
              $ref: "#/$defs/JsonSchemaVersions"
    properties:
      format:
        description: The format targeted by this rule
        type: string
        enum:
          - openapi
          - asyncapi
          - jsonschema
      versions:
        description: The version of the format targeted by this rule
        type: array
        minItems: 1
        uniqueItems: true
      severity:
        description: The severity of the problems returned by the rule
        type: string
        enum:
          - error
          - warn
          - info
          - hint
      recommended:
        description: Is the rule recommended (activated)?
        type: boolean
        default: true
      given:
        description: |
          The list of test checking the JSON Path Plus path(s) provided in given:
          
          - find something and the value is what is expected (minimal, at least one of those)
          - do not find what is supposed to ignore (for complex path involving filters)
        type: array
        items:
          $ref: "#/$defs/GivenTest"
      then:
        description: |
          The list of tests checking the rule returns the expected problems or not.
          There must be at least a test checking problems are returned, and test without problems.
        type: array
        items:
          $ref: "#/$defs/ThenTest"
  GivenTest:
    description: The description of a given's JSON Path Plus test
    type: object
    unevaluatedProperties: false
    required:
      - description
      - expected
      - documents
    properties:
      description:
        description: The description of the test
        type: string
      expected:
        description: The expected result of the test. Hence, the values and their paths which are supposed to be found. 
        type: array
        items:
          $ref: "#/$defs/SpectralPath"
      documents:
        $ref: "#/$defs/Documents"
  ThenTest:
    description: The description of a test checking the problems returned when executing this rule.
    type: object
    unevaluatedProperties: false
    required:
      - description
      - expected
      - documents
    properties:
      description:
        description: The description of the test
        type: string
      expected:
        description: |
          The expected result of the test. Hence, the Spectral paths of the problems detected.
          Its value is an empty array if provided documents are valid. 
        type: array
        minItems: 0
        items:
          $ref: "#/$defs/SpectralPath"
      documents:
        $ref: "#/$defs/Documents"
  SpectralPath:
    description: The Spectral path targeting the found element (as returned by Spectral).
    type: array
    items:
      oneOf:
        - type: string
        - type: integer
    minItems: 1
  Documents:
    description: A list of or a single test document
    oneOf:
      - $ref: "#/$defs/Document"
      - type: array
        minItems: 1
        uniqueItems: true
        items:
          $ref: "#/$defs/Document"
  Document:
    description: |
      A document that will be used as template to find what is expected across the versions.
      If no version is provided (direct specification document, or no `versions`` provided), 
      the document is considered as a template, and documents matching rule test versions 
      will be generated.
    oneOf:
      - $ref: "#/$defs/SpecificationDocument"
      - type: object
        required:
          - document
        unevaluatedProperties: false
        properties:
          versions:
            description: The versions to use to generate the documents
            oneOf:
              - $ref: "#/$defs/VersionNumber"
              - type: array
                minItems: 1
                uniqueItems: true
                items:
                  $ref: "#/$defs/VersionNumber"
          document:
            description: Will be used as a template if more than one versions are provided 
            $ref: "#/$defs/SpecificationDocument"
  SpecificationDocument:
    type: object
    oneOf:
      - required: [swagger]
      - required: [openapi]
      - required: [asyncapi]