Merge pull request #1321 from ajv-validator/dynamic-ref

Dynamic recursive references

docs/api.md

@@ -259,6 +259,7 @@ const defaultOptions = {
   // validation and reporting options:
   next: false,
   unevaluated: false,
+  dynamicRef: false,
   $data: false,
   allErrors: false,
   verbose: false,
@@ -317,6 +318,7 @@ const defaultOptions = {
 
 - _next_: add support for the keywords from the next JSON-Schema draft (currently it is draft 2019-09): [`dependentRequired`](./json-schema.md#dependentrequired), [`dependentSchemas`](./json-schema.md#dependentschemas), [`maxContains`/`minContain`](./json-schema.md#maxcontains--mincontains). This option will be removed once the next draft is fully supported.
 - _unevaluated_: to track evaluated properties/items and support keywords [`unevaluatedProperties`](./json-schema.md#unevaluatedproperties) and [`unevaluatedItems`](./json-schema.md#unevaluateditems). Supporting these keywords may add additional validation-time logic even to validation functions where these keywords are not used. When possible, Ajv determines which properties/items are "unevaluated" at compilation time.
+- _dynamicRef_: to support `recursiveRef`/`recursiveAnchor` keywords (JSON Schema draft-2019-09) and `dynamicRef`/`dynamicAnchor` keywords (the upcoming JSON Schema draft). See [Extending recursive schemas](./validation.md#extending-recursive-schemas)
 - _\$data_: support [\$data references](./validation.md#data-reference). Draft 6 meta-schema that is added by default will be extended to allow them. If you want to use another meta-schema you need to use $dataMetaSchema method to add support for $data reference. See [API](#ajv-constructor-and-methods).
 - _allErrors_: check all rules collecting all errors. Default is to return after the first error.
 - _verbose_: include the reference to the part of the schema (`schema` and `parentSchema`) and validated data in errors (false by default).

docs/validation.md

@@ -7,6 +7,7 @@
     - [Formats](#formats)
   - [Modular schemas](#modular-schemas)
     - [<a name="ref"></a>Combining schemas with \$ref](#combining-schemas-with-ref)
+    - [Extending recursive schemas](#extending-recursive-schemas)
     - [\$data reference](#data-reference)
     - [$merge and $patch keywords](#merge-and-patch-keywords)
   - [User-defined keywords](#user-defined-keywords)
@@ -135,6 +136,74 @@ See [Options](./api.md#options) and [addSchema](./api.md#add-schema) method.
 - You cannot have the same \$id (or the schema identifier) used for more than one schema - the exception will be thrown.
 - You can implement dynamic resolution of the referenced schemas using `compileAsync` method. In this way you can store schemas in any system (files, web, database, etc.) and reference them without explicitly adding to Ajv instance. See [Asynchronous schema compilation](./validation.md#asynchronous-schema-compilation).
 
+### Extending recursive schemas
+
+While statically defined `$ref` keyword allows to split schemas to multiple files, it is difficult to extend recursive schemas - the recursive reference(s) in the original schema points to the original schema, and not to the extended one. So in JSON Schema draft-07 the only available solution to extend the recursive schema was to redifine all sections of the original schema that have recursion.
+
+It was particularly repetitive when extending meta-schema, as it has many recursive references, but even in a schema with a single recursive reference extending it was very verbose.
+
+JSON Schema draft-2019-09 and the upcoming draft defined the mechanism for dynamic recursion using keywords `recursiveRef`/`recursiveAnchor` (draft-2019-09) or `dynamicRef`/`dynamicAnchor` (the next JSON Schema draft) that is somewhat similar to "open recursion" in functional programming.
+
+Consider this recursive schema with static recursion:
+
+```javascript
+const treeSchema = {
+  $id: "https://example.com/tree",
+  type: "object",
+  required: ["data"],
+  properties: {
+    data: true,
+    children: {
+      type: "array",
+      items: {$ref: "#"},
+    },
+  },
+}
+```
+
+The only way to extend this schema to prohibit additional properties is by adding `additionalProperties` keyword right in the schema - this approach can be impossible if you do not control the source of the original schema. Ajv also provided the additional keywords in [ajv-merge-patch](https://github.com/ajv-validator/ajv-merge-patch) package to extend schemas by treating them as plain JSON data. While this approach works, it is non-standard.
+
+The new keywords for dynamic recursive references allow extending this schema without modifying it:
+
+```javascript
+const treeSchema = {
+  $id: "https://example.com/tree",
+  $recursiveAnchor: true,
+  type: "object",
+  required: ["data"],
+  properties: {
+    data: true,
+    children: {
+      type: "array",
+      items: {$recursiveRef: "#"},
+    },
+  },
+}
+
+const strictTreeSchema = {
+  $id: "https://example.com/strict-tree",
+  $recursiveAnchor: true,
+  $ref: "tree",
+  unevaluatedProperties: false,
+}
+
+const ajv = new Ajv({
+  dynamicRef: true, // to support dynamic recursive references
+  unevaluated: true, // to support unevaluatedProperties
+  schemas: [treeSchema, strictTreeSchema],
+})
+const validate = ajv.getSchema("https://example.com/strict-tree")
+```
+
+See [dynamic-refs](../spec/dynamic-ref.spec.ts) test for the example using `$dynamicAnchor`/`$dynamicRef`.
+
+At the moment Ajv implements the spec for dynamic recursive references with these limitations:
+
+- `$recursiveAnchor`/`$dynamicAnchor` can only be used in the schema root.
+- `$recursiveRef`/`$dynamicRef` can only be hash fragments, without URI.
+
+Ajv also does not support dynamic references in [asynchronous schemas](#asynchronous-validation) (Ajv spec extension), it is assumed that the referenced schema is synchronous - there is no validation-time check.
+
 ### \$data reference
 
 With `$data` option you can use values from the validated data as the values for the schema keywords. See [proposal](https://github.com/json-schema-org/json-schema-spec/issues/51) for more information about how it works.

lib/ajv.ts

@@ -61,6 +61,7 @@ import formatVocabulary from "./vocabularies/format"
 import {metadataVocabulary, contentVocabulary} from "./vocabularies/metadata"
 import nextVocabulary from "./vocabularies/next"
 import unevaluatedVocabulary from "./vocabularies/unevaluated"
+import dynamicVocabulary from "./vocabularies/dynamic"
 import {eachItem} from "./compile/util"
 import $dataRefSchema from "./refs/data.json"
 import draft7MetaSchema from "./refs/json-schema-draft-07.json"
@@ -95,13 +96,14 @@ interface CurrentOptions {
   // validation and reporting options:
   next?: boolean
   unevaluated?: boolean
+  dynamicRef?: boolean
   $data?: boolean
   allErrors?: boolean
   verbose?: boolean
   $comment?:
     | true
     | ((comment: string, schemaPath?: string, rootSchema?: AnySchemaObject) => unknown)
-  formats?: {[name: string]: Format}
+  formats?: {[Name in string]?: Format}
   keywords?: Vocabulary
   schemas?: AnySchema[] | {[key: string]: AnySchema}
   logger?: Logger | false
@@ -273,6 +275,7 @@ export default class Ajv {
 
     if (opts.formats) addInitialFormats.call(this)
     this.addVocabulary(["$async"])
+    if (opts.dynamicRef) this.addVocabulary(dynamicVocabulary)
     this.addVocabulary(coreVocabulary)
     this.addVocabulary(validationVocabulary)
     this.addVocabulary(applicatorVocabulary)
@@ -731,7 +734,7 @@ function addInitialSchemas(this: Ajv): void {
 function addInitialFormats(this: Ajv): void {
   for (const name in this.opts.formats) {
     const format = this.opts.formats[name]
-    this.addFormat(name, format)
+    if (format) this.addFormat(name, format)
   }
 }
 

lib/compile/index.ts

@@ -41,6 +41,7 @@ export interface SchemaCxt {
   readonly strictSchema?: boolean
   readonly rootId: string
   baseId: string // the current schema base URI that should be used as the base for resolving URIs in references (\$ref)
+  dynamicAnchors: {[Ref in string]?: true}
   readonly schemaPath: Code // the run-time expression that evaluates to the property name of the current schema
   readonly errSchemaPath: string // this is actual string, should not be changed to Code
   readonly errorPath: Code
@@ -132,6 +133,7 @@ export function compileSchema(this: Ajv, sch: SchemaEnv): SchemaEnv {
     strictSchema: true,
     rootId,
     baseId: sch.baseId || rootId,
+    dynamicAnchors: {},
     schemaPath: nil,
     errSchemaPath: "#",
     errorPath: _`""`,
@@ -190,7 +192,7 @@ export function resolveRef(
   root: SchemaEnv,
   baseId: string,
   ref: string
-): AnySchema | AnyValidateFunction | SchemaEnv | undefined {
+): AnySchema | SchemaEnv | undefined {
   ref = resolveUrl(baseId, ref)
   const schOrFunc = root.refs[ref]
   if (schOrFunc) return schOrFunc

lib/compile/names.ts

@@ -4,11 +4,12 @@ const names = {
   // validation function arguments
   data: new Name("data"), // data passed to validation function
   // args passed from referencing schema
-  dataCxt: new Name("dataCxt"), // should not be used directly, it is destructured to the names below
+  valCxt: new Name("valCxt"), // validation/data context - should not be used directly, it is destructured to the names below
   dataPath: new Name("dataPath"),
   parentData: new Name("parentData"),
   parentDataProperty: new Name("parentDataProperty"),
   rootData: new Name("rootData"), // root data - same as the data passed to the first/top validation function
+  dynamicAnchors: new Name("dynamicAnchors"), // used to support recursiveRef and dynamicRef
   // function scoped variables
   vErrors: new Name("vErrors"), // null or array of validation errors
   errors: new Name("errors"), // counter of validation errors

lib/compile/resolve.ts

@@ -37,9 +37,17 @@ export function inlineRef(schema: AnySchema, limit: boolean | number = true): bo
   return countKeys(schema) <= limit
 }
 
+const REF_KEYWORDS = new Set([
+  "$ref",
+  "$recursiveRef",
+  "$recursiveAnchor",
+  "$dynamicRef",
+  "$dynamicAnchor",
+])
+
 function hasRef(schema: AnySchemaObject): boolean {
   for (const key in schema) {
-    if (key === "$ref") return true
+    if (REF_KEYWORDS.has(key)) return true
     const sch = schema[key]
     if (Array.isArray(sch) && sch.some(hasRef)) return true
     if (typeof sch == "object" && hasRef(sch)) return true
@@ -81,41 +89,59 @@ export function resolveUrl(baseId: string, id: string): string {
   return URI.resolve(baseId, id)
 }
 
+const ANCHOR = /^[a-z_][-a-z0-9._]*$/i
+
 export function getSchemaRefs(this: Ajv, schema: AnySchema): LocalRefs {
   if (typeof schema == "boolean") return {}
   const schemaId = normalizeId(schema.$id)
   const baseIds: {[jsonPtr: string]: string} = {"": schemaId}
   const pathPrefix = getFullPath(schemaId, false)
   const localRefs: LocalRefs = {}
+  const schemaRefs: Set<string> = new Set()
 
   traverse(schema, {allKeys: true}, (sch, jsonPtr, _, parentJsonPtr) => {
     if (parentJsonPtr === undefined) return
     const fullPath = pathPrefix + jsonPtr
-    let id = sch.$id
     let baseId = baseIds[parentJsonPtr]
-    if (typeof id == "string") {
-      id = baseId = normalizeId(baseId ? URI.resolve(baseId, id) : id)
-      let schOrRef = this.refs[id]
+    if (typeof sch.$id == "string") baseId = addRef.call(this, sch.$id)
+    addAnchor.call(this, sch.$anchor)
+    addAnchor.call(this, sch.$dynamicAnchor)
+    baseIds[jsonPtr] = baseId
+
+    function addRef(this: Ajv, ref: string): string {
+      ref = normalizeId(baseId ? URI.resolve(baseId, ref) : ref)
+      if (schemaRefs.has(ref)) throw ambiguos(ref)
+      schemaRefs.add(ref)
+      let schOrRef = this.refs[ref]
       if (typeof schOrRef == "string") schOrRef = this.refs[schOrRef]
       if (typeof schOrRef == "object") {
-        checkAmbiguosId(sch, schOrRef.schema, id)
-      } else if (id !== normalizeId(fullPath)) {
-        if (id[0] === "#") {
-          checkAmbiguosId(sch, localRefs[id], id)
-          localRefs[id] = sch
+        checkAmbiguosRef(sch, schOrRef.schema, ref)
+      } else if (ref !== normalizeId(fullPath)) {
+        if (ref[0] === "#") {
+          checkAmbiguosRef(sch, localRefs[ref], ref)
+          localRefs[ref] = sch
         } else {
-          this.refs[id] = fullPath
+          this.refs[ref] = fullPath
         }
       }
+      return ref
+    }
+
+    function addAnchor(this: Ajv, anchor: unknown): void {
+      if (typeof anchor == "string") {
+        if (!ANCHOR.test(anchor)) throw new Error(`invalid anchor "${anchor}"`)
+        addRef.call(this, `#${anchor}`)
+      }
     }
-    baseIds[jsonPtr] = baseId
   })
 
   return localRefs
 
-  function checkAmbiguosId(sch1: AnySchema, sch2: AnySchema | undefined, id: string): void {
-    if (sch2 !== undefined && !equal(sch1, sch2)) {
-      throw new Error(`id "${id}" resolves to more than one schema`)
-    }
+  function checkAmbiguosRef(sch1: AnySchema, sch2: AnySchema | undefined, ref: string): void {
+    if (sch2 !== undefined && !equal(sch1, sch2)) throw ambiguos(ref)
+  }
+
+  function ambiguos(ref: string): Error {
+    return new Error(`reference "${ref}" resolves to more than one schema`)
   }
 }

lib/compile/rules.ts

@@ -41,7 +41,7 @@ export function getRules(): ValidationRules {
   }
   return {
     types: {...groups, integer: true, boolean: true, null: true},
-    rules: [groups.number, groups.string, {rules: []}, groups.array, groups.object],
+    rules: [{rules: []}, groups.number, groups.string, groups.array, groups.object],
     all: {type: true, $comment: true},
     keywords: {type: true, $comment: true},
   }

lib/compile/validate/index.ts

@@ -27,34 +27,39 @@ function validateFunction(
 ): void {
   gen.return(() =>
     opts.code.es5
-      ? gen.func(validateName, _`${N.data}, ${N.dataCxt}`, schemaEnv.$async, () => {
+      ? gen.func(validateName, _`${N.data}, ${N.valCxt}`, schemaEnv.$async, () => {
           gen.code(_`"use strict"; ${funcSourceUrl(schema, opts)}`)
-          destructureDataCxtES5(gen)
+          destructureValCxtES5(gen, opts)
           gen.code(body)
         })
-      : gen.func(
-          validateName,
-          _`${N.data}, {${N.dataPath}="", ${N.parentData}, ${N.parentDataProperty}, ${N.rootData}=${N.data}}={}`,
-          schemaEnv.$async,
-          () => gen.code(funcSourceUrl(schema, opts)).code(body)
+      : gen.func(validateName, _`${N.data}, ${destructureValCxt(opts)}`, schemaEnv.$async, () =>
+          gen.code(funcSourceUrl(schema, opts)).code(body)
         )
   )
 }
 
-function destructureDataCxtES5(gen: CodeGen): void {
+function destructureValCxt(opts: InstanceOptions): Code {
+  return _`{${N.dataPath}="", ${N.parentData}, ${N.parentDataProperty}, ${N.rootData}=${N.data}${
+    opts.dynamicRef ? _`, ${N.dynamicAnchors}={}` : nil
+  }}={}`
+}
+
+function destructureValCxtES5(gen: CodeGen, opts: InstanceOptions): void {
   gen.if(
-    N.dataCxt,
+    N.valCxt,
     () => {
-      gen.var(N.dataPath, _`${N.dataCxt}.${N.dataPath}`)
-      gen.var(N.parentData, _`${N.dataCxt}.${N.parentData}`)
-      gen.var(N.parentDataProperty, _`${N.dataCxt}.${N.parentDataProperty}`)
-      gen.var(N.rootData, _`${N.dataCxt}.${N.rootData}`)
+      gen.var(N.dataPath, _`${N.valCxt}.${N.dataPath}`)
+      gen.var(N.parentData, _`${N.valCxt}.${N.parentData}`)
+      gen.var(N.parentDataProperty, _`${N.valCxt}.${N.parentDataProperty}`)
+      gen.var(N.rootData, _`${N.valCxt}.${N.rootData}`)
+      if (opts.dynamicRef) gen.var(N.dynamicAnchors, _`${N.valCxt}.${N.dynamicAnchors}`)
     },
     () => {
       gen.var(N.dataPath, _`""`)
       gen.var(N.parentData, _`undefined`)
       gen.var(N.parentDataProperty, _`undefined`)
       gen.var(N.rootData, N.data)
+      if (opts.dynamicRef) gen.var(N.dynamicAnchors, _`{}`)
     }
   )
 }

lib/refs/json-schema-2019-09/index.ts

@@ -0,0 +1,27 @@
+import type Ajv from "../../ajv"
+import type {SchemaObject} from "../../ajv"
+import metaSchema from "./schema.json"
+import metaApplicator from "./meta/applicator.json"
+import metaContent from "./meta/content.json"
+import metaCore from "./meta/core.json"
+import metaFormat from "./meta/format.json"
+import metaMetadata from "./meta/meta-data.json"
+import metaValidation from "./meta/validation.json"
+
+export default function addMetaSchema2019(ajv: Ajv, setDefault?: boolean): Ajv {
+  ;[
+    metaSchema,
+    metaApplicator,
+    metaContent,
+    metaCore,
+    metaFormat,
+    metaMetadata,
+    metaValidation,
+  ].forEach(addMeta)
+  if (setDefault) ajv.opts.defaultMeta = "https://json-schema.org/draft/2019-09/schema"
+  return ajv
+
+  function addMeta(sch: SchemaObject): void {
+    ajv.addMetaSchema(sch, undefined, false)
+  }
+}