-
Notifications
You must be signed in to change notification settings - Fork 46
perf: Add schema caching to parameter validation #261
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
6038995
526d925
7271dd3
97ddd74
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -13,11 +13,13 @@ import ( | |
| "github.com/pb33f/libopenapi/datamodel/high/base" | ||
| "github.com/pb33f/libopenapi/utils" | ||
| "github.com/santhosh-tekuri/jsonschema/v6" | ||
| "go.yaml.in/yaml/v4" | ||
| "golang.org/x/text/language" | ||
| "golang.org/x/text/message" | ||
|
|
||
| stdError "errors" | ||
|
|
||
| "github.com/pb33f/libopenapi-validator/cache" | ||
| "github.com/pb33f/libopenapi-validator/config" | ||
| "github.com/pb33f/libopenapi-validator/errors" | ||
| "github.com/pb33f/libopenapi-validator/helpers" | ||
|
|
@@ -35,16 +37,52 @@ func ValidateSingleParameterSchema( | |
| pathTemplate string, | ||
| operation string, | ||
| ) (validationErrors []*errors.ValidationError) { | ||
| // Get the JSON Schema for the parameter definition. | ||
| jsonSchema, err := buildJsonRender(schema) | ||
| if err != nil { | ||
| return validationErrors | ||
| var jsch *jsonschema.Schema | ||
| var jsonSchema []byte | ||
|
|
||
| // Try cache lookup first - avoids expensive schema compilation on each request | ||
| if o != nil && o.SchemaCache != nil && schema != nil && schema.GoLow() != nil { | ||
| hash := schema.GoLow().Hash() | ||
| if cached, ok := o.SchemaCache.Load(hash); ok && cached != nil && cached.CompiledSchema != nil { | ||
| jsch = cached.CompiledSchema | ||
| } | ||
| } | ||
|
|
||
| // Attempt to compile the JSON Schema | ||
| jsch, err := helpers.NewCompiledSchema(name, jsonSchema, o) | ||
| if err != nil { | ||
| return validationErrors | ||
| // Cache miss - compile the schema | ||
| if jsch == nil { | ||
| // Get the JSON Schema for the parameter definition. | ||
| var err error | ||
| jsonSchema, err = buildJsonRender(schema) | ||
| if err != nil { | ||
| return validationErrors | ||
| } | ||
|
|
||
| // Attempt to compile the JSON Schema | ||
| jsch, err = helpers.NewCompiledSchema(name, jsonSchema, o) | ||
| if err != nil { | ||
| return validationErrors | ||
| } | ||
|
|
||
| // Store in cache for future requests | ||
| if o != nil && o.SchemaCache != nil && schema != nil && schema.GoLow() != nil { | ||
| hash := schema.GoLow().Hash() | ||
|
|
||
| renderCtx := base.NewInlineRenderContextForValidation() | ||
| renderedInline, _ := schema.RenderInlineWithContext(renderCtx) | ||
| referenceSchema := string(renderedInline) | ||
|
|
||
| var renderedNode yaml.Node | ||
| _ = yaml.Unmarshal(renderedInline, &renderedNode) | ||
|
|
||
| o.SchemaCache.Store(hash, &cache.SchemaCacheEntry{ | ||
| Schema: schema, | ||
| RenderedInline: renderedInline, | ||
| ReferenceSchema: referenceSchema, | ||
| RenderedJSON: jsonSchema, | ||
| CompiledSchema: jsch, | ||
| RenderedNode: &renderedNode, | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| // Validate the object and report any errors. | ||
|
|
@@ -71,6 +109,27 @@ func buildJsonRender(schema *base.Schema) ([]byte, error) { | |
| return utils.ConvertYAMLtoJSON(renderedSchema) | ||
| } | ||
|
|
||
| // GetRenderedSchema returns a YAML string representation of the schema for error messages. | ||
| // It first checks the schema cache for a pre-rendered version, falling back to fresh rendering. | ||
| // This avoids expensive re-rendering on each validation when the cache is available. | ||
| func GetRenderedSchema(schema *base.Schema, opts *config.ValidationOptions) string { | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This still leaves Cache entries are written at line 70 and 170, but cache misses fall back to In The same missing/parse error can still emit different |
||
| if schema == nil { | ||
| return "" | ||
| } | ||
|
|
||
| // Try cache lookup first | ||
| if opts != nil && opts.SchemaCache != nil && schema.GoLow() != nil { | ||
| hash := schema.GoLow().Hash() | ||
| if cached, ok := opts.SchemaCache.Load(hash); ok && cached != nil && len(cached.RenderedInline) > 0 { | ||
| return string(cached.RenderedInline) | ||
| } | ||
| } | ||
|
|
||
| // Cache miss - render fresh as YAML | ||
| rendered, _ := schema.RenderInline() | ||
| return string(rendered) | ||
| } | ||
|
|
||
| // ValidateParameterSchema will validate a parameter against a raw object, or a blob of json/yaml. | ||
| // It will return a list of validation errors, if any. | ||
| // | ||
|
|
@@ -94,13 +153,64 @@ func ValidateParameterSchema( | |
| validationOptions *config.ValidationOptions, | ||
| ) []*errors.ValidationError { | ||
| var validationErrors []*errors.ValidationError | ||
| var jsch *jsonschema.Schema | ||
| var jsonSchema []byte | ||
|
|
||
| // Try cache lookup first - avoids expensive schema compilation on each request | ||
| if validationOptions != nil && validationOptions.SchemaCache != nil && schema != nil && schema.GoLow() != nil { | ||
| hash := schema.GoLow().Hash() | ||
| if cached, ok := validationOptions.SchemaCache.Load(hash); ok && cached != nil && cached.CompiledSchema != nil { | ||
| jsch = cached.CompiledSchema | ||
| } | ||
| } | ||
|
|
||
| // Cache miss - render and compile the schema | ||
| if jsch == nil { | ||
| // 1. build a JSON render of the schema. | ||
| renderCtx := base.NewInlineRenderContextForValidation() | ||
| renderedSchema, _ := schema.RenderInlineWithContext(renderCtx) | ||
| referenceSchema := string(renderedSchema) | ||
| jsonSchema, _ = utils.ConvertYAMLtoJSON(renderedSchema) | ||
|
|
||
| // 1. build a JSON render of the schema. | ||
| renderCtx := base.NewInlineRenderContextForValidation() | ||
| renderedSchema, _ := schema.RenderInlineWithContext(renderCtx) | ||
| jsonSchema, _ := utils.ConvertYAMLtoJSON(renderedSchema) | ||
| // 2. create a new json schema compiler and add the schema to it | ||
| var err error | ||
| jsch, err = helpers.NewCompiledSchema(name, jsonSchema, validationOptions) | ||
| if err != nil { | ||
| // schema compilation failed, return validation error instead of panicking | ||
| validationErrors = append(validationErrors, &errors.ValidationError{ | ||
| ValidationType: validationType, | ||
| ValidationSubType: subValType, | ||
| Message: fmt.Sprintf("%s '%s' failed schema compilation", entity, name), | ||
| Reason: fmt.Sprintf("%s '%s' schema compilation failed: %s", | ||
| reasonEntity, name, err.Error()), | ||
| SpecLine: 1, | ||
| SpecCol: 0, | ||
| ParameterName: name, | ||
| HowToFix: "check the parameter schema for invalid JSON Schema syntax, complex regex patterns, or unsupported schema constructs", | ||
| Context: string(jsonSchema), | ||
| }) | ||
| return validationErrors | ||
| } | ||
|
|
||
| // Store in cache for future requests | ||
| if validationOptions != nil && validationOptions.SchemaCache != nil && schema != nil && schema.GoLow() != nil { | ||
| hash := schema.GoLow().Hash() | ||
|
|
||
| // 2. decode the object into a json blob. | ||
| var renderedNode yaml.Node | ||
| _ = yaml.Unmarshal(renderedSchema, &renderedNode) | ||
|
|
||
| validationOptions.SchemaCache.Store(hash, &cache.SchemaCacheEntry{ | ||
| Schema: schema, | ||
| RenderedInline: renderedSchema, | ||
| ReferenceSchema: referenceSchema, | ||
| RenderedJSON: jsonSchema, | ||
| CompiledSchema: jsch, | ||
| RenderedNode: &renderedNode, | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| // 3. decode the object into a json blob. | ||
| var decodedObj interface{} | ||
| rawIsMap := false | ||
| validEncoding := false | ||
|
|
@@ -125,24 +235,6 @@ func ValidateParameterSchema( | |
| } | ||
| validEncoding = true | ||
| } | ||
| // 3. create a new json schema compiler and add the schema to it | ||
| jsch, err := helpers.NewCompiledSchema(name, jsonSchema, validationOptions) | ||
| if err != nil { | ||
| // schema compilation failed, return validation error instead of panicking | ||
| validationErrors = append(validationErrors, &errors.ValidationError{ | ||
| ValidationType: validationType, | ||
| ValidationSubType: subValType, | ||
| Message: fmt.Sprintf("%s '%s' failed schema compilation", entity, name), | ||
| Reason: fmt.Sprintf("%s '%s' schema compilation failed: %s", | ||
| reasonEntity, name, err.Error()), | ||
| SpecLine: 1, | ||
| SpecCol: 0, | ||
| ParameterName: name, | ||
| HowToFix: "check the parameter schema for invalid JSON Schema syntax, complex regex patterns, or unsupported schema constructs", | ||
| Context: string(jsonSchema), | ||
| }) | ||
| return validationErrors | ||
| } | ||
|
|
||
| // 4. validate the object against the schema | ||
| var scErrs error | ||
|
|
@@ -241,8 +333,7 @@ func formatJsonSchemaValidationError(schema *base.Schema, scErrs *jsonschema.Val | |
| renderCtx := base.NewInlineRenderContextForValidation() | ||
| rendered, err := schema.RenderInlineWithContext(renderCtx) | ||
| if err == nil && rendered != nil { | ||
| renderedBytes, _ := json.Marshal(rendered) | ||
| fail.ReferenceSchema = string(renderedBytes) | ||
| fail.ReferenceSchema = string(rendered) | ||
|
daveshanley marked this conversation as resolved.
|
||
| } | ||
| } | ||
| schemaValidationErrors = append(schemaValidationErrors, fail) | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.