fix(schema): deduplicate JSON Schema fallback definitions

.changeset/deduplicate-json-schema-fallbacks.md

@@ -0,0 +1,10 @@
+---
+"effect": patch
+"@effect/openapi-generator": patch
+---
+
+Deduplicate equivalent fallback definitions when compiling JSON Schema, and reconstruct only definitions reachable from multi-document roots.
+
+Remove `SchemaMultiDocument` and `fromSchemaMultiDocument`; multi-document import and revival now return the ordered root schemas directly.
+
+Stop the OpenAPI generator from emitting component schemas that are not reachable from a generated root.

packages/effect/SCHEMA.md

@@ -6185,7 +6185,6 @@ flowchart TD
     LD -->|toJsonSchemaDocument|JD["JsonSchema.Document (draft-2020-12)"]
     JD -->|fromJsonSchemaDocument|S
     LD -->|toMultiDocument|LMD["live MultiDocument"]
-    SMD[SchemaMultiDocument] -->|fromSchemaMultiDocument|LMD
     LMD -->|toCodeDocument|CodeDocument
     LMD -->|toJsonSchemaMultiDocument|JMD[JsonSchema.MultiDocument]
     LMD -->|toJsonMultiDocument|JSON
@@ -6213,20 +6212,6 @@ A `MultiDocument` stores multiple root representations that share the same `refe
 
 This is useful if you want to serialize a set of schemas together, or if you want to generate code for multiple schemas while emitting shared definitions only once.
 
-### `SchemaMultiDocument`
-
-A `SchemaMultiDocument` contains live schemas plus a named definition map:
-
-```ts
-interface SchemaMultiDocument {
-  readonly schemas: readonly [Schema.Top, ...Array<Schema.Top>]
-  readonly definitions: Readonly<Record<string, Schema.Top>>
-}
-```
-
-`fromJsonSchemaMultiDocument` returns this form. `fromSchemaMultiDocument` projects it to a `MultiDocument` while
-preserving explicit definitions, including definitions that are not reachable from a root.
-
 ## Projection and persistence boundaries
 
 ### Representations use the encoded side
@@ -6374,7 +6359,8 @@ Effect exports individual revivers next to the built-in declarations and checks
 `Schema.OptionReviver`, `Schema.DateReviver`, and `Schema.isMinLengthReviver`. Supply every reviver required by the
 document; a missing or duplicate `id`, or a payload that does not satisfy its reviver's `payloadSchema`, is an error.
 
-`fromRepresentations` rebuilds every root and named definition in a `MultiDocument` and returns a `SchemaMultiDocument`.
+`fromRepresentations` rebuilds the ordered roots of a `MultiDocument` in a shared reference environment. Only references
+reachable from those roots are revived.
 
 ### Custom revivers
 
@@ -6430,8 +6416,8 @@ schema with revivers first.
 `SchemaRepresentation.fromJsonSchemaDocument` imports a JSON Schema Draft 2020-12 document as a runtime `Schema.Top`.
 It does not return a representation document.
 
-`fromJsonSchemaMultiDocument` returns a `SchemaMultiDocument` containing all root schemas and definitions. Use
-`fromSchemaMultiDocument` when that result must be passed to a representation compiler.
+`fromJsonSchemaMultiDocument` returns the ordered root schemas. It translates only definitions reachable from those
+roots. To pass the result to a representation compiler, call `toRepresentations` with the returned schemas' ASTs.
 
 Import is best-effort: JSON Schema constructs are translated to Effect schemas where possible, but the result is not a
 lossless reconstruction of an original Effect schema. The optional `onEnter` callback can normalize each JSON Schema node

packages/effect/src/SchemaRepresentation.ts

@@ -487,17 +487,6 @@ export interface MultiDocument {
   readonly references: References
 }
 
-/**
- * Live schemas reconstructed from a multi-document.
- *
- * @category models
- * @since 4.0.0
- */
-export interface SchemaMultiDocument {
-  readonly schemas: readonly [Schema.Top, ...Array<Schema.Top>]
-  readonly definitions: Readonly<Record<string, Schema.Top>>
-}
-
 /**
  * Reviver for a declaration.
  *
@@ -713,27 +702,6 @@ export function toRepresentations(
   return InternalToRepresentation.toRepresentations(asts)
 }
 
-/**
- * Converts live schemas and their named definitions to a shared representation document.
- *
- * **When to use**
- *
- * Use when schemas with shared or unreachable definitions must be passed to representation compilers such as `toCodeDocument`.
- *
- * **Gotchas**
- *
- * Every schema is projected to its encoded side. Definitions are preserved even when no root reaches them.
- *
- * @see {@link toRepresentations} for converting AST roots without an explicit definition map
- * @see {@link toCodeDocument} for generating code from the result
- *
- * @category constructors
- * @since 4.0.0
- */
-export function fromSchemaMultiDocument(document: SchemaMultiDocument): MultiDocument {
-  return InternalToRepresentation.fromSchemaMultiDocument(document)
-}
-
 /**
  * Wraps a single representation document as a multi-document with one root.
  *
@@ -1134,15 +1102,15 @@ export function fromRepresentation(
 }
 
 /**
- * Reconstructs multiple runtime schemas and their shared definitions from a representation multi-document.
+ * Reconstructs multiple runtime schemas from a representation multi-document.
  *
  * **When to use**
  *
- * Use when every root and named definition must be rebuilt in one shared reference environment.
+ * Use when multiple roots must be rebuilt in one shared reference environment.
  *
  * **Gotchas**
  *
- * Every definition is revived, including definitions not reachable from a root. Revivers are resolved locally by `id`; none are installed implicitly.
+ * Only references reachable from a root are revived. Revivers are resolved locally by `id`; none are installed implicitly.
  *
  * @see {@link fromJsonMultiDocument} for decoding a persisted multi-document
  * @see {@link fromRepresentation} for a single root
@@ -1153,7 +1121,7 @@ export function fromRepresentation(
 export function fromRepresentations(
   document: MultiDocument,
   options: { readonly revivers: ReadonlyArray<AnyReviver> }
-): SchemaMultiDocument {
+): readonly [Schema.Top, ...Array<Schema.Top>] {
   return InternalFromRepresentation.fromRepresentations(document, options.revivers)
 }
 
@@ -1186,21 +1154,21 @@ export function fromJsonSchemaDocument(
  *
  * **When to use**
  *
- * Use when multiple imported roots must preserve shared definitions, aliases, and recursion.
+ * Use when multiple imported roots share reachable definitions, aliases, or recursion.
  *
  * **Gotchas**
  *
- * Every definition is translated, including definitions that no root references. Callback results are used directly, and exceptions raised by a callback pass through unchanged.
+ * Only definitions reachable from a root are translated. Callback results are used directly, and exceptions raised by a callback pass through unchanged.
  *
  * @see {@link fromJsonSchemaDocument} for a single root
- * @see {@link fromSchemaMultiDocument} for converting the result to a representation document
+ * @see {@link toRepresentations} for converting the returned schema ASTs to a representation document
  *
  * @category constructors
  * @since 4.0.0
  */
 export function fromJsonSchemaMultiDocument(
   document: JsonSchema.MultiDocument<"draft-2020-12">,
   options?: FromJsonSchemaOptions
-): SchemaMultiDocument {
+): readonly [Schema.Top, ...Array<Schema.Top>] {
   return InternalFromJsonSchemaDocument.fromJsonSchemaMultiDocument(document, options)
 }

packages/effect/src/internal/schema/fromJsonSchemaDocument.ts

@@ -214,8 +214,8 @@ function translateJsonSchemaMultiDocument(
   options?: SchemaRepresentation.FromJsonSchemaOptions,
   singleRoot = false
 ): SchemaRepresentation.MultiDocument {
-  const definitionCache = new Map<string, ImportedJsonSchemaRepresentation>()
-  const definitionsInProgress = new Set<string>()
+  const definitionCache = new Map<string, ImportedJsonSchemaRepresentation | null>()
+  const reachableDefinitions = new Map<string, Path>()
   const annotatedReferences: Array<{
     readonly reference: SchemaRepresentation.Reference
     readonly path: Path
@@ -227,16 +227,17 @@ function translateJsonSchemaMultiDocument(
     recursiveReferenceError?: string
   ): ImportedJsonSchemaRepresentation {
     const cached = definitionCache.get(key)
-    if (cached !== undefined) return cached
+    if (cached !== undefined) {
+      if (cached === null) {
+        throw errorWithPath(recursiveReferenceError ?? `Invalid reference ${key}`, [...path, "$ref"])
+      }
+      return cached
+    }
     if (!Object.hasOwn(document.definitions, key)) {
       throw errorWithPath(`Invalid reference ${key}`, [...path, "$ref"])
     }
-    if (definitionsInProgress.has(key)) {
-      throw errorWithPath(recursiveReferenceError ?? `Invalid reference ${key}`, [...path, "$ref"])
-    }
-    definitionsInProgress.add(key)
+    definitionCache.set(key, null)
     const representation = recur(document.definitions[key], ["definitions", key])
-    definitionsInProgress.delete(key)
     definitionCache.set(key, representation)
     return representation
   }
@@ -696,6 +697,7 @@ function translateJsonSchemaMultiDocument(
     if (typeof schema.$ref === "string") {
       const $ref = jsonSchemaReferenceKey(schema.$ref)
       if ($ref !== undefined) {
+        if (!reachableDefinitions.has($ref)) reachableDefinitions.set($ref, path)
         return { _tag: "Reference", $ref }
       }
     }
@@ -890,12 +892,12 @@ function translateJsonSchemaMultiDocument(
   }
 
   const references: Record<string, Representation> = {}
-  for (const key of Object.keys(document.definitions)) {
-    InternalRecord.assignProperty(references, key, unknownJsonSchemas(translateDefinition(key, ["definitions", key])))
-  }
   const representations = document.schemas.map((schema, index) =>
     unknownJsonSchemas(recur(schema, singleRoot ? ["schema"] : ["schemas", index]))
   ) as [Representation, ...Array<Representation>]
+  for (const [key, path] of reachableDefinitions) {
+    InternalRecord.assignProperty(references, key, unknownJsonSchemas(translateDefinition(key, path)))
+  }
   for (const { reference, path } of annotatedReferences) {
     resolveReference(reference, path)
   }
@@ -952,6 +954,6 @@ export function fromJsonSchemaDocument(
 export function fromJsonSchemaMultiDocument(
   document: JsonSchema.MultiDocument<"draft-2020-12">,
   options?: SchemaRepresentation.FromJsonSchemaOptions
-): SchemaRepresentation.SchemaMultiDocument {
+): readonly [Schema.Top, ...Array<Schema.Top>] {
   return fromRepresentations(translateJsonSchemaMultiDocument(document, options), jsonSchemaRevivers)
 }

packages/effect/src/internal/schema/fromRepresentation.ts

@@ -12,7 +12,7 @@ type Path = ReadonlyArray<string | number>
 export function fromRepresentations(
   document: SchemaRepresentation.MultiDocument,
   revivers: ReadonlyArray<SchemaRepresentation.AnyReviver>
-): SchemaRepresentation.SchemaMultiDocument {
+): readonly [Schema.Top, ...Array<Schema.Top>] {
   return revivePersisted(document.representations, document.references, makeReviverMap(revivers), false)
 }
 
@@ -58,18 +58,17 @@ function revivePersisted(
   references: SchemaRepresentation.References,
   reviverMap: ReadonlyMap<string, SchemaRepresentation.AnyReviver>,
   singleRoot: boolean
-): SchemaRepresentation.SchemaMultiDocument {
+): readonly [Schema.Top, ...Array<Schema.Top>] {
   const slots = new Map<string, ReferenceSlot>()
-  const referenceKeys = Object.keys(references)
-
-  for (const key of referenceKeys) {
-    slots.set(key, new ReferenceSlot(key))
-  }
 
   function resolveReference(key: string, path: Path): Schema.Top {
-    const slot = slots.get(key)
+    let slot = slots.get(key)
     if (slot === undefined) {
-      throw errorWithPath(`Invalid reference ${key}`, [...path, "$ref"])
+      if (!Object.hasOwn(references, key)) {
+        throw errorWithPath(`Invalid reference ${key}`, [...path, "$ref"])
+      }
+      slot = new ReferenceSlot(key)
+      slots.set(key, slot)
     }
     if (slot.body !== undefined) {
       return slot.body
@@ -320,15 +319,10 @@ function revivePersisted(
     }
   }
 
-  const definitions: Record<string, Schema.Top> = {}
-  for (const key of referenceKeys) {
-    InternalRecord.assignProperty(definitions, key, resolveReference(key, ["references", key]))
-  }
-
   const schemas = representations.map((representation, index) =>
     recur(representation, singleRoot ? ["representation"] : ["representations", index])
   ) as [Schema.Top, ...Array<Schema.Top>]
-  return { schemas, definitions }
+  return schemas
 }
 
 /** @internal */
@@ -341,5 +335,5 @@ export function fromRepresentation(
     document.references,
     makeReviverMap(revivers),
     true
-  ).schemas[0]
+  )[0]
 }

packages/effect/src/internal/schema/toJsonSchemaDocument.ts

@@ -1,4 +1,5 @@
 import * as Arr from "../../Array.ts"
+import * as Equal from "../../Equal.ts"
 import { escapeToken } from "../../JsonPointer.ts"
 import type * as JsonSchema from "../../JsonSchema.ts"
 import * as RegEx from "../../RegExp.ts"
@@ -142,12 +143,63 @@ function compileJsonSchema(
   options: Schema.ToJsonSchemaOptions | undefined
 ): JsonSchema.MultiDocument<"draft-2020-12"> {
   const definitions: Record<string, JsonSchema.JsonSchema> = {}
-  for (const key of Object.keys(references)) {
-    InternalRecord.assignProperty(definitions, key, recur(references[key], ["references", key]))
+  // null = compiling, string = canonical key, object = compiled schema
+  const definitionStates = new Map<string, JsonSchema.JsonSchema | string | null>()
+  const compiledRepresentations = new WeakMap<SchemaRepresentation.Representation, JsonSchema.JsonSchema>()
+  const fallbackDefinitions = new Map<string, Array<string>>()
+  const referenceKeys = Object.keys(references)
+  for (const key of referenceKeys) {
+    compileDefinition(key, ["references", key])
+  }
+  for (const key of referenceKeys) {
+    const compiled = definitionStates.get(key)!
+    if (typeof compiled !== "string") {
+      InternalRecord.assignProperty(definitions, key, compiled)
+    }
   }
   const schemas = Arr.map(representations, (representation, index) => recur(representation, rootPaths[index]))
   return { dialect: "draft-2020-12", schemas, definitions }
 
+  function compileDefinition(key: string, path: Path): string {
+    const compiled = definitionStates.get(key)
+    if (compiled !== undefined) return typeof compiled === "string" ? compiled : key
+    if (!Object.hasOwn(references, key)) {
+      throw errorWithPath(`Invalid reference ${key}`, [...path, "$ref"])
+    }
+
+    definitionStates.set(key, null)
+    const representation = references[key]
+    const schema = recur(representation, ["references", key])
+
+    const fallback = getIdentifierFallback(representation)
+    if (fallback !== undefined) {
+      const candidates = fallbackDefinitions.get(fallback)
+      const match = candidates?.find((candidate) => Equal.equals(definitionStates.get(candidate), schema))
+      if (match === undefined) {
+        if (candidates === undefined) fallbackDefinitions.set(fallback, [key])
+        else candidates.push(key)
+      } else {
+        definitionStates.set(key, match)
+        return match
+      }
+    }
+    definitionStates.set(key, schema)
+    return key
+  }
+
+  function getIdentifierFallback(
+    representation: SchemaRepresentation.Representation
+  ): string | undefined {
+    if (representation._tag === "Reference") return undefined
+    const annotations = representation.checks.length === 0
+      ? representation.annotations
+      : representation.checks[representation.checks.length - 1].annotations
+    return typeof annotations?.identifier !== "string" &&
+        typeof annotations?.[InternalAnnotations.IDENTIFIER_FALLBACK_KEY] === "string"
+      ? annotations[InternalAnnotations.IDENTIFIER_FALLBACK_KEY]
+      : undefined
+  }
+
   function annotationSchemas(
     representation: CheckRepresentationAnnotation | undefined,
     path: Path
@@ -183,11 +235,11 @@ function compileJsonSchema(
     path: Path
   ): JsonSchema.JsonSchema {
     if (representation._tag === "Reference") {
-      if (!Object.hasOwn(references, representation.$ref)) {
-        throw errorWithPath(`Invalid reference ${representation.$ref}`, [...path, "$ref"])
-      }
-      return { $ref: `#/$defs/${escapeToken(representation.$ref)}` }
+      const canonical = compileDefinition(representation.$ref, path)
+      return { $ref: `#/$defs/${escapeToken(canonical)}` }
     }
+    const cached = compiledRepresentations.get(representation)
+    if (cached !== undefined) return cached
 
     let output = on(representation, path)
     const ordinary = collectJsonSchemaAnnotations(representation.annotations, options)
@@ -201,6 +253,7 @@ function compileJsonSchema(
         output = appendJsonSchema(output, check)
       }
     }
+    compiledRepresentations.set(representation, output)
     return output
   }
 
@@ -370,6 +423,7 @@ function compileJsonSchema(
         if (!Object.hasOwn(references, parameter.$ref)) {
           throw errorWithPath(`Invalid reference ${parameter.$ref}`, [...path, "$ref"])
         }
+        compileDefinition(parameter.$ref, path)
         if (seenReferences.has(parameter.$ref)) return []
         const next = new Set(seenReferences).add(parameter.$ref)
         return getParameterPatterns(references[parameter.$ref], ["references", parameter.$ref], next)