diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/event_envelope.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/event_envelope.rb index 605024146..18ab6561f 100644 --- a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/event_envelope.rb +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/event_envelope.rb @@ -7,78 +7,82 @@ # frozen_string_literal: true require "elastic_graph/constants" +require "elastic_graph/json_ingestion" module ElasticGraph - module SchemaDefinition - module Indexing - # Contains logic related to "event envelope"--the layer of metadata that wraps all indexing events. - # - # @api private - module EventEnvelope - # @param indexed_type_names [Array] names of the indexed types - # @param json_schema_version [Integer] the version of the JSON schema - # @return [Hash] the JSON schema for the ElasticGraph event envelope for the given `indexed_type_names`. - def self.json_schema(indexed_type_names, json_schema_version) - { - "type" => "object", - "description" => "Required by ElasticGraph to wrap every data event.", - "properties" => { - "op" => { - "description" => "Indicates what type of operation the event represents. For now, only `upsert` is supported, but we plan to support other operations in the future.", - "type" => "string", - "enum" => %w[upsert] - }, - "type" => { - "description" => "The type of object present in `record`.", - "type" => "string", - # Sorting doesn't really matter here, but it's nice for the output in the schema artifact to be consistent. - "enum" => indexed_type_names.sort - }, - "id" => { - "description" => "The unique identifier of the record.", - "type" => "string", - "maxLength" => DEFAULT_MAX_KEYWORD_LENGTH - }, - "version" => { - "description" => 'Used to handle duplicate and out-of-order events. When ElasticGraph ingests multiple events for the same `type` and `id`, the one with the largest `version` will "win".', - "type" => "integer", - "minimum" => 0, - "maximum" => (2**63) - 1 - }, - "record" => { - "description" => "The record of this event. The payload of this field must match the JSON schema of the named `type`.", - "type" => "object" - }, - "latency_timestamps" => { - "description" => "Timestamps from which ElasticGraph measures indexing latency. The `ElasticGraphIndexingLatencies` log message produced for each event will include a measurement from each timestamp included in this map.", - "type" => "object", - "additionalProperties" => false, - "patternProperties" => { - "^\\w+_at$" => { - "description" => "A timestamp from which ElasticGraph will measure indexing latency. The timestamp name must end in `_at`.", - "type" => "string", - "format" => "date-time" + module JSONIngestion + module SchemaDefinition + # Indexing support used while generating JSON ingestion schemas. + module Indexing + # Contains logic related to "event envelope"--the layer of metadata that wraps all indexing events. + # + # @api private + module EventEnvelope + # @param indexed_type_names [Array] names of the indexed types + # @param json_schema_version [Integer] the version of the JSON schema + # @return [Hash] the JSON schema for the ElasticGraph event envelope for the given `indexed_type_names`. + def self.json_schema(indexed_type_names, json_schema_version) + { + "type" => "object", + "description" => "Required by ElasticGraph to wrap every data event.", + "properties" => { + "op" => { + "description" => "Indicates what type of operation the event represents. For now, only `upsert` is supported, but we plan to support other operations in the future.", + "type" => "string", + "enum" => %w[upsert] + }, + "type" => { + "description" => "The type of object present in `record`.", + "type" => "string", + # Sorting doesn't really matter here, but it's nice for the output in the schema artifact to be consistent. + "enum" => indexed_type_names.sort + }, + "id" => { + "description" => "The unique identifier of the record.", + "type" => "string", + "maxLength" => ElasticGraph::DEFAULT_MAX_KEYWORD_LENGTH + }, + "version" => { + "description" => 'Used to handle duplicate and out-of-order events. When ElasticGraph ingests multiple events for the same `type` and `id`, the one with the largest `version` will "win".', + "type" => "integer", + "minimum" => 0, + "maximum" => (2**63) - 1 + }, + "record" => { + "description" => "The record of this event. The payload of this field must match the JSON schema of the named `type`.", + "type" => "object" + }, + "latency_timestamps" => { + "description" => "Timestamps from which ElasticGraph measures indexing latency. The `ElasticGraphIndexingLatencies` log message produced for each event will include a measurement from each timestamp included in this map.", + "type" => "object", + "additionalProperties" => false, + "patternProperties" => { + "^\\w+_at$" => { + "description" => "A timestamp from which ElasticGraph will measure indexing latency. The timestamp name must end in `_at`.", + "type" => "string", + "format" => "date-time" + } } + }, + ElasticGraph::JSON_SCHEMA_VERSION_KEY => { + "description" => "The version of the JSON schema the publisher was using when the event was published. ElasticGraph will use the JSON schema matching this version to process the event.", + "const" => json_schema_version + }, + "message_id" => { + "description" => "The optional ID of the message containing this event from whatever messaging system is being used between the publisher and the ElasticGraph indexer.", + "type" => "string" } }, - JSON_SCHEMA_VERSION_KEY => { - "description" => "The version of the JSON schema the publisher was using when the event was published. ElasticGraph will use the JSON schema matching this version to process the event.", - "const" => json_schema_version + "additionalProperties" => false, + "required" => ["op", "type", "id", "version", ElasticGraph::JSON_SCHEMA_VERSION_KEY], + "if" => { + "properties" => { + "op" => {"const" => "upsert"} + } }, - "message_id" => { - "description" => "The optional ID of the message containing this event from whatever messaging system is being used between the publisher and the ElasticGraph indexer.", - "type" => "string" - } - }, - "additionalProperties" => false, - "required" => ["op", "type", "id", "version", JSON_SCHEMA_VERSION_KEY], - "if" => { - "properties" => { - "op" => {"const" => "upsert"} - } - }, - "then" => {"required" => ["record"]} - } + "then" => {"required" => ["record"]} + } + end end end end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_field_metadata.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_field_metadata.rb index 535d11b2d..83c94b9d4 100644 --- a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_field_metadata.rb +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_field_metadata.rb @@ -7,27 +7,29 @@ # frozen_string_literal: true module ElasticGraph - module SchemaDefinition - module Indexing - # @!parse class JSONSchemaFieldMetadata; end - JSONSchemaFieldMetadata = ::Data.define(:type, :name_in_index) + module JSONIngestion + module SchemaDefinition + module Indexing + # @!parse class JSONSchemaFieldMetadata; end + JSONSchemaFieldMetadata = ::Data.define(:type, :name_in_index) - # Metadata about an ElasticGraph field that needs to be stored in our versioned JSON schemas - # alongside the JSON schema fields. - # - # @!attribute [r] type - # @return [String] name of the ElasticGraph type for this field - # @!attribute [r] name_in_index - # @return [String] name of the field in the index - # - # @api private - class JSONSchemaFieldMetadata < ::Data - # @return [Hash] hash form of the metadata that can be dumped in JSON schema - def to_dumpable_hash - {"type" => type, "nameInIndex" => name_in_index} - end + # Metadata about an ElasticGraph field that needs to be stored in our versioned JSON schemas + # alongside the JSON schema fields. + # + # @!attribute [r] type + # @return [String] name of the ElasticGraph type for this field + # @!attribute [r] name_in_index + # @return [String] name of the field in the index + # + # @api private + class JSONSchemaFieldMetadata < ::Data + # @return [Hash] hash form of the metadata that can be dumped in JSON schema + def to_dumpable_hash + {"type" => type, "nameInIndex" => name_in_index} + end - # @dynamic initialize, type, name_in_index + # @dynamic initialize, type, name_in_index + end end end end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rb index a56d9c2e7..94076d440 100644 --- a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rb +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rb @@ -7,227 +7,242 @@ # frozen_string_literal: true require "elastic_graph/constants" +require "elastic_graph/json_ingestion" module ElasticGraph - module SchemaDefinition - module Indexing - # Represents the result of merging a JSON schema with metadata. The result includes both - # the merged JSON schema and a list of `failed_fields` indicating which fields metadata - # could not be determined for. - # - # @private - class JSONSchemaWithMetadata < ::Data.define( - # The JSON schema. - :json_schema, - # A set of fields (in the form `Type.field`) that were needed but not found. - :missing_fields, - # A set of type names that were needed but not found. - :missing_types, - # A set of `DeprecatedElement` objects that create conflicting definitions. - :definition_conflicts, - # A set of fields that have been deleted but that must be retained (e.g. for custom shard routing or rollover) - :missing_necessary_fields - ) - def json_schema_version - json_schema.fetch(JSON_SCHEMA_VERSION_KEY) - end - - # Responsible for building `JSONSchemaWithMetadata` instances. + module JSONIngestion + module SchemaDefinition + module Indexing + # Represents the result of merging a JSON schema with ElasticGraph metadata. + # The result includes the merged JSON schema plus details about schema elements + # from prior versions that could not be matched to current metadata. + # + # @!attribute [r] json_schema + # @return [Hash] the merged JSON schema + # @!attribute [r] missing_fields + # @return [Set] fields (in the form `Type.field`) that were needed but not found + # @!attribute [r] missing_types + # @return [Set] type names that were needed but not found + # @!attribute [r] definition_conflicts + # @return [Set] deprecated elements that create conflicting definitions + # @!attribute [r] missing_necessary_fields + # @return [Array] deleted fields that must be retained for routing or rollover # # @private - class Merger - # @dynamic unused_deprecated_elements - attr_reader :unused_deprecated_elements - - def initialize(schema_def_results) - @field_metadata_by_type_and_field_name = schema_def_results.json_schema_field_metadata_by_type_and_field_name - @renamed_types_by_old_name = schema_def_results.state.renamed_types_by_old_name - @deleted_types_by_old_name = schema_def_results.state.deleted_types_by_old_name - @renamed_fields_by_type_name_and_old_field_name = schema_def_results.state.renamed_fields_by_type_name_and_old_field_name - @deleted_fields_by_type_name_and_old_field_name = schema_def_results.state.deleted_fields_by_type_name_and_old_field_name - @state = schema_def_results.state - @derived_indexing_type_names = schema_def_results.derived_indexing_type_names - - @unused_deprecated_elements = ( - @renamed_types_by_old_name.values + - @deleted_types_by_old_name.values + - @renamed_fields_by_type_name_and_old_field_name.values.flat_map(&:values) + - @deleted_fields_by_type_name_and_old_field_name.values.flat_map(&:values) - ).to_set + class JSONSchemaWithMetadata < ::Data.define( + # The JSON schema. + :json_schema, + # A set of fields (in the form `Type.field`) that were needed but not found. + :missing_fields, + # A set of type names that were needed but not found. + :missing_types, + # A set of `DeprecatedElement` objects that create conflicting definitions. + :definition_conflicts, + # A set of fields that have been deleted but that must be retained (e.g. for custom shard routing or rollover) + :missing_necessary_fields + ) + def json_schema_version + json_schema.fetch(ElasticGraph::JSON_SCHEMA_VERSION_KEY) end - def merge_metadata_into(json_schema) - missing_fields = ::Set.new - missing_types = ::Set.new - definition_conflicts = ::Set.new - old_type_name_by_current_name = {} # : ::Hash[String, String] - - defs = json_schema.fetch("$defs").to_h do |type_name, type_def| - if type_name != EVENT_ENVELOPE_JSON_SCHEMA_NAME && (properties = type_def["properties"]) - current_type_name = determine_current_type_name( - type_name, - missing_types: missing_types, - definition_conflicts: definition_conflicts - ) - - if current_type_name - old_type_name_by_current_name[current_type_name] = type_name - end + # Responsible for building `JSONSchemaWithMetadata` instances. + # + # @private + class Merger + # @dynamic unused_deprecated_elements + attr_reader :unused_deprecated_elements + + def initialize(schema_def_results) + @field_metadata_by_type_and_field_name = schema_def_results.json_schema_field_metadata_by_type_and_field_name + @renamed_types_by_old_name = schema_def_results.state.renamed_types_by_old_name + @deleted_types_by_old_name = schema_def_results.state.deleted_types_by_old_name + @renamed_fields_by_type_name_and_old_field_name = schema_def_results.state.renamed_fields_by_type_name_and_old_field_name + @deleted_fields_by_type_name_and_old_field_name = schema_def_results.state.deleted_fields_by_type_name_and_old_field_name + @state = schema_def_results.state + @derived_indexing_type_names = schema_def_results.derived_indexing_type_names + + @unused_deprecated_elements = ( + @renamed_types_by_old_name.values + + @deleted_types_by_old_name.values + + @renamed_fields_by_type_name_and_old_field_name.values.flat_map(&:values) + + @deleted_fields_by_type_name_and_old_field_name.values.flat_map(&:values) + ).to_set + end - properties = properties.to_h do |field_name, prop| - unless field_name == "__typename" - field_metadata = current_type_name&.then do |name| - field_metadata_for( - name, - field_name, - missing_fields: missing_fields, - definition_conflicts: definition_conflicts - ) + def merge_metadata_into(json_schema) + missing_fields = ::Set.new + missing_types = ::Set.new + definition_conflicts = ::Set.new + old_type_name_by_current_name = {} # : ::Hash[::String, ::String] + + defs = json_schema.fetch("$defs").to_h do |type_name, type_def| + if type_name != ElasticGraph::EVENT_ENVELOPE_JSON_SCHEMA_NAME && (properties = type_def["properties"]) + current_type_name = determine_current_type_name( + type_name, + missing_types: missing_types, + definition_conflicts: definition_conflicts + ) + + if current_type_name + old_type_name_by_current_name[current_type_name] = type_name + end + + properties = properties.to_h do |field_name, prop| + unless field_name == "__typename" + field_metadata = current_type_name&.then do |name| + field_metadata_for( + name, + field_name, + missing_fields: missing_fields, + definition_conflicts: definition_conflicts + ) + end + + prop = prop.merge({"ElasticGraph" => field_metadata&.to_dumpable_hash}) end - prop = prop.merge({"ElasticGraph" => field_metadata&.to_dumpable_hash}) + [field_name, prop] end - [field_name, prop] + type_def = type_def.merge({"properties" => properties}) end - type_def = type_def.merge({"properties" => properties}) + [type_name, type_def] end - [type_name, type_def] + json_schema = json_schema.merge("$defs" => defs) + + JSONSchemaWithMetadata.new( + json_schema: json_schema, + missing_fields: missing_fields, + missing_types: missing_types, + definition_conflicts: definition_conflicts, + missing_necessary_fields: identify_missing_necessary_fields(json_schema, old_type_name_by_current_name) + ) end - json_schema = json_schema.merge("$defs" => defs) + private - JSONSchemaWithMetadata.new( - json_schema: json_schema, - missing_fields: missing_fields, - missing_types: missing_types, - definition_conflicts: definition_conflicts, - missing_necessary_fields: identify_missing_necessary_fields(json_schema, old_type_name_by_current_name) - ) - end + # Given a historical `type_name`, determines (and returns) the current name for that type. + def determine_current_type_name(type_name, missing_types:, definition_conflicts:) + exists_currently = @field_metadata_by_type_and_field_name.key?(type_name) + deleted = @deleted_types_by_old_name[type_name]&.tap { |elem| @unused_deprecated_elements.delete(elem) } + renamed = @renamed_types_by_old_name[type_name]&.tap { |elem| @unused_deprecated_elements.delete(elem) } - private + if [exists_currently, deleted, renamed].count(&:itself) > 1 + definition_conflicts.merge([deleted, renamed].compact) + end - # Given a historical `type_name`, determines (and returns) the current name for that type. - def determine_current_type_name(type_name, missing_types:, definition_conflicts:) - exists_currently = @field_metadata_by_type_and_field_name.key?(type_name) - deleted = @deleted_types_by_old_name[type_name]&.tap { |elem| @unused_deprecated_elements.delete(elem) } - renamed = @renamed_types_by_old_name[type_name]&.tap { |elem| @unused_deprecated_elements.delete(elem) } + return type_name if exists_currently + return nil if deleted + return renamed.name if renamed - if [exists_currently, deleted, renamed].count(&:itself) > 1 - definition_conflicts.merge([deleted, renamed].compact) + missing_types << type_name + nil end - return type_name if exists_currently - return nil if deleted - return renamed.name if renamed + # Given a historical `type_name` and `field_name` determines (and returns) the field metadata for it. + def field_metadata_for(type_name, field_name, missing_fields:, definition_conflicts:) + full_name = "#{type_name}.#{field_name}" - missing_types << type_name - nil - end + current_meta = @field_metadata_by_type_and_field_name.dig(type_name, field_name) + deleted = @deleted_fields_by_type_name_and_old_field_name.dig(type_name, field_name)&.tap do |elem| + @unused_deprecated_elements.delete(elem) + end + renamed = @renamed_fields_by_type_name_and_old_field_name.dig(type_name, field_name)&.tap do |elem| + @unused_deprecated_elements.delete(elem) + end - # Given a historical `type_name` and `field_name` determines (and returns) the field metadata for it. - def field_metadata_for(type_name, field_name, missing_fields:, definition_conflicts:) - full_name = "#{type_name}.#{field_name}" + if [current_meta, deleted, renamed].count(&:itself) > 1 + definition_conflicts.merge([deleted, renamed].compact.map { |elem| elem.with(name: full_name) }) + end - current_meta = @field_metadata_by_type_and_field_name.dig(type_name, field_name) - deleted = @deleted_fields_by_type_name_and_old_field_name.dig(type_name, field_name)&.tap do |elem| - @unused_deprecated_elements.delete(elem) - end - renamed = @renamed_fields_by_type_name_and_old_field_name.dig(type_name, field_name)&.tap do |elem| - @unused_deprecated_elements.delete(elem) - end + return current_meta if current_meta + return nil if deleted + return @field_metadata_by_type_and_field_name.dig(type_name, renamed.name) if renamed - if [current_meta, deleted, renamed].count(&:itself) > 1 - definition_conflicts.merge([deleted, renamed].compact.map { |elem| elem.with(name: full_name) }) + missing_fields << full_name + nil end - return current_meta if current_meta - return nil if deleted - return @field_metadata_by_type_and_field_name.dig(type_name, renamed.name) if renamed - - missing_fields << full_name - nil - end + def identify_missing_necessary_fields(json_schema, old_type_name_by_current_name) + json_schema_resolver = JSONSchemaResolver.new(@state, json_schema, old_type_name_by_current_name) - def identify_missing_necessary_fields(json_schema, old_type_name_by_current_name) - json_schema_resolver = JSONSchemaResolver.new(@state, json_schema, old_type_name_by_current_name) - version = json_schema.fetch(JSON_SCHEMA_VERSION_KEY) - - @state.object_types_by_name.values - .select { |type| type.has_own_index_def? && !@derived_indexing_type_names.include?(type.name) } - .flat_map do |object_type| - identify_missing_necessary_fields_for_index_def( - object_type, - object_type.own_index_def, # : Indexing::Index - json_schema_resolver, version - ) - end - end + @state.object_types_by_name.values + .select { |type| type.has_own_index_def? && !@derived_indexing_type_names.include?(type.name) } + .flat_map do |object_type| + index_def = object_type.own_index_def # : ElasticGraph::SchemaDefinition::Indexing::Index + identify_missing_necessary_fields_for_index_def(object_type, index_def, json_schema_resolver) + end + end - def identify_missing_necessary_fields_for_index_def(object_type, index_def, json_schema_resolver, json_schema_version) - { - "routing" => index_def.routing_field_path, - "rollover" => index_def.rollover_config&.timestamp_field_path - }.compact.filter_map do |field_type, field_path| - if json_schema_resolver.necessary_path_missing?(field_path) - # The JSON schema v # {json_schema_version} artifact has no field that maps to the #{field_type} path of `#{field_path.fully_qualified_path_in_index}`. - - MissingNecessaryField.new( - field_type: field_type, - fully_qualified_path: field_path.fully_qualified_path_in_index - ) + def identify_missing_necessary_fields_for_index_def(object_type, index_def, json_schema_resolver) + { + "routing" => index_def.routing_field_path, + "rollover" => index_def.rollover_config&.timestamp_field_path + }.compact.filter_map do |field_type, field_path| + if json_schema_resolver.necessary_path_missing?(field_path) + # The JSON schema artifact has no field that maps to the #{field_type} path of `#{field_path.fully_qualified_path_in_index}`. + MissingNecessaryField.new( + field_type: field_type, + fully_qualified_path: field_path.fully_qualified_path_in_index + ) + end end end - end - class JSONSchemaResolver - def initialize(state, json_schema, old_type_name_by_current_name) - @state = state - @old_type_name_by_current_name = old_type_name_by_current_name - @meta_by_old_type_and_name_in_index = ::Hash.new do |hash, type_name| - properties = json_schema.fetch("$defs").fetch(type_name).fetch("properties") + class JSONSchemaResolver + def initialize(state, json_schema, old_type_name_by_current_name) + @state = state + @old_type_name_by_current_name = old_type_name_by_current_name + @meta_by_old_type_and_name_in_index = ::Hash.new do |hash, type_name| + properties = json_schema.fetch("$defs").fetch(type_name).fetch("properties") - hash[type_name] = properties.filter_map do |name, prop| - if (metadata = prop["ElasticGraph"]) - [metadata.fetch("nameInIndex"), metadata] - end - end.to_h + hash[type_name] = properties.filter_map do |name, prop| + if (metadata = prop["ElasticGraph"]) + [metadata.fetch("nameInIndex"), metadata] + end + end.to_h + end end - end - # Indicates if the given `field_path` is (1) necessary and (2) missing from the JSON schema, indicating a problem. - # - # - Returns `false` is the given `field_path` is present in the JSON schema. - # - Returns `false` is the parent type of `field_path` has not been retained in this JSON schema version - # (in that case, the field path is not necessary). - # - Otherwise, returns `true` since the field path is both necessary and missing. - def necessary_path_missing?(field_path) - parent_type = field_path.first_part.parent_type.name - - field_path.path_parts.any? do |path_part| - necessary_path_part_missing?(parent_type, path_part.name_in_index) do |meta| - parent_type = @state.type_ref(meta.fetch("type")).fully_unwrapped.name + # Indicates if the given `field_path` is (1) necessary and (2) missing from the JSON schema, indicating a problem. + # + # - Returns `false` is the given `field_path` is present in the JSON schema. + # - Returns `false` is the parent type of `field_path` has not been retained in this JSON schema version + # (in that case, the field path is not necessary). + # - Otherwise, returns `true` since the field path is both necessary and missing. + def necessary_path_missing?(field_path) + parent_type = field_path.first_part.parent_type.name + + field_path.path_parts.any? do |path_part| + necessary_path_part_missing?(parent_type, path_part.name_in_index) do |meta| + parent_type = @state.type_ref(meta.fetch("type")).fully_unwrapped.name + end end end - end - private + private - def necessary_path_part_missing?(parent_type, name_in_index) - old_type_name = @old_type_name_by_current_name[parent_type] - return false unless old_type_name + def necessary_path_part_missing?(parent_type, name_in_index) + old_type_name = @old_type_name_by_current_name[parent_type] + return false unless old_type_name - meta = @meta_by_old_type_and_name_in_index.dig(old_type_name, name_in_index) - yield meta if meta - !meta + meta = @meta_by_old_type_and_name_in_index.dig(old_type_name, name_in_index) + yield meta if meta + !meta + end end end - end - MissingNecessaryField = ::Data.define(:field_type, :fully_qualified_path) + # @!parse class MissingNecessaryField < ::Data; end + MissingNecessaryField = ::Data.define(:field_type, :fully_qualified_path) + + # @private + class MissingNecessaryField < ::Data + # @dynamic initialize, with, field_type, fully_qualified_path + end + end end end end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/json_schema_pruner.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/json_schema_pruner.rb index 7a8323fa6..850f25b3f 100644 --- a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/json_schema_pruner.rb +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/json_schema_pruner.rb @@ -7,54 +7,56 @@ # frozen_string_literal: true require "elastic_graph/constants" +require "elastic_graph/json_ingestion" module ElasticGraph - module SchemaDefinition - # Prunes unused type definitions from a given JSON schema. - # - # @private - class JSONSchemaPruner - def self.prune(original_json_schema) - initial_type_names = [EVENT_ENVELOPE_JSON_SCHEMA_NAME] + original_json_schema - .dig("$defs", EVENT_ENVELOPE_JSON_SCHEMA_NAME, "properties", "type", "enum") - - types_to_keep = referenced_type_names(initial_type_names, original_json_schema["$defs"]) - - # The .select will preserve the sort order of the original hash - # standard:disable Style/HashSlice -- https://github.com/soutaro/steep/issues/1503 - pruned_defs = original_json_schema["$defs"].select { |k, _v| types_to_keep.include?(k) } - # standard:enable Style/HashSlice - - original_json_schema.merge("$defs" => pruned_defs) - end + module JSONIngestion + # Schema definition integration for JSON ingestion artifacts. + module SchemaDefinition + # Prunes unused type definitions from a given JSON schema. + # + # @private + class JSONSchemaPruner + def self.prune(original_json_schema) + initial_type_names = [ElasticGraph::EVENT_ENVELOPE_JSON_SCHEMA_NAME] + original_json_schema + .dig("$defs", ElasticGraph::EVENT_ENVELOPE_JSON_SCHEMA_NAME, "properties", "type", "enum") - # Returns a list of type names indicating all types referenced from any type in source_type_names. - private_class_method - def self.referenced_type_names(source_type_names, original_defs) - return Set.new if source_type_names.empty? + types_to_keep = referenced_type_names(initial_type_names, original_json_schema["$defs"]) - referenced_type_defs = original_defs.slice(*source_type_names) - ref_names = collect_ref_names(referenced_type_defs) + # The .select will preserve the sort order of the original hash + # standard:disable Style/HashSlice -- https://github.com/soutaro/steep/issues/1503 + pruned_defs = original_json_schema["$defs"].select { |type_name, _type_def| types_to_keep.include?(type_name) } + # standard:enable Style/HashSlice - referenced_type_names(ref_names, original_defs) + source_type_names - end + original_json_schema.merge("$defs" => pruned_defs) + end + + # Returns a list of type names indicating all types referenced from any type in source_type_names. + private_class_method def self.referenced_type_names(source_type_names, original_defs) + return Set.new if source_type_names.empty? + + referenced_type_defs = original_defs.slice(*source_type_names) + ref_names = collect_ref_names(referenced_type_defs) + + referenced_type_names(ref_names, original_defs) + source_type_names + end - private_class_method - def self.collect_ref_names(hash) - hash.flat_map do |key, value| - case value - when ::Hash - collect_ref_names(value) - when ::Array - value.grep(::Hash).flat_map { |subhash| collect_ref_names(subhash) } - when ::String - if key == "$ref" && (type = value[%r{\A#/\$defs/(.+)\z}, 1]) - [type] + private_class_method def self.collect_ref_names(hash) + hash.flat_map do |key, value| + case value + when ::Hash + collect_ref_names(value) + when ::Array + value.grep(::Hash).flat_map { |subhash| collect_ref_names(subhash) } + when ::String + if key == "$ref" && (type = value[%r{\A#/\$defs/(.+)\z}, 1]) + [type] + else + [] + end else [] end - else - [] end end end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/event_envelope.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/event_envelope.rbs index aa609f3de..bed69cb23 100644 --- a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/event_envelope.rbs +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/event_envelope.rbs @@ -1,8 +1,10 @@ module ElasticGraph - module SchemaDefinition - module Indexing - module EventEnvelope - def self.json_schema: (::Array[::String], ::Integer) -> ::Hash[::String, untyped] + module JSONIngestion + module SchemaDefinition + module Indexing + module EventEnvelope + def self.json_schema: (::Array[::String], ::Integer) -> ::Hash[::String, untyped] + end end end end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_field_metadata.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_field_metadata.rbs index b025eda15..98948fbca 100644 --- a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_field_metadata.rbs +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_field_metadata.rbs @@ -1,16 +1,18 @@ module ElasticGraph - module SchemaDefinition - module Indexing - class JSONSchemaFieldMetadata - attr_reader type: ::String - attr_reader name_in_index: ::String + module JSONIngestion + module SchemaDefinition + module Indexing + class JSONSchemaFieldMetadata + attr_reader type: ::String + attr_reader name_in_index: ::String - def initialize: ( - type: ::String, - name_in_index: ::String - ) -> void + def initialize: ( + type: ::String, + name_in_index: ::String + ) -> void - def to_dumpable_hash: () -> {"type" => ::String, "nameInIndex" => ::String} + def to_dumpable_hash: () -> {"type" => ::String, "nameInIndex" => ::String} + end end end end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rbs index dcc37b607..a553ed9a8 100644 --- a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rbs +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rbs @@ -1,96 +1,102 @@ module ElasticGraph - module SchemaDefinition - module Indexing - class JSONSchemaWithMetadataSupertype - attr_reader json_schema: ::Hash[::String, untyped] - attr_reader missing_fields: ::Set[::String] - attr_reader missing_types: ::Set[::String] - attr_reader definition_conflicts: ::Set[SchemaElements::DeprecatedElement] - attr_reader missing_necessary_fields: ::Array[JSONSchemaWithMetadata::MissingNecessaryField] - - def initialize: ( - json_schema: ::Hash[::String, untyped], - missing_fields: ::Set[::String], - missing_types: ::Set[::String], - definition_conflicts: ::Set[SchemaElements::DeprecatedElement], - missing_necessary_fields: ::Array[JSONSchemaWithMetadata::MissingNecessaryField] - ) -> void - - def with: ( - ?json_schema: ::Hash[::String, untyped], - ?missing_fields: ::Set[::String], - ?missing_types: ::Set[::String], - ?definition_conflicts: ::Set[SchemaElements::DeprecatedElement], - ?missing_necessary_fields: ::Array[JSONSchemaWithMetadata::MissingNecessaryField] - ) -> instance - end - - class JSONSchemaWithMetadata < JSONSchemaWithMetadataSupertype - def json_schema_version: () -> ::Integer - - class Merger - @field_metadata_by_type_and_field_name: ::Hash[::String, ::Hash[::String, JSONSchemaFieldMetadata]] - @renamed_types_by_old_name: ::Hash[::String, SchemaElements::DeprecatedElement] - @deleted_types_by_old_name: ::Hash[::String, SchemaElements::DeprecatedElement] - @renamed_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]] - @deleted_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]] - @state: State - @derived_indexing_type_names: ::Set[::String] - - attr_reader unused_deprecated_elements: ::Set[SchemaElements::DeprecatedElement] + module JSONIngestion + module SchemaDefinition + module Indexing + class JSONSchemaWithMetadataSupertype + attr_reader json_schema: ::Hash[::String, untyped] + attr_reader missing_fields: ::Set[::String] + attr_reader missing_types: ::Set[::String] + attr_reader definition_conflicts: ::Set[ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement] + attr_reader missing_necessary_fields: ::Array[JSONSchemaWithMetadata::MissingNecessaryField] - def initialize: (Results) -> void - def merge_metadata_into: (::Hash[::String, untyped]) -> JSONSchemaWithMetadata - - private - - def determine_current_type_name: ( - ::String, + def initialize: ( + json_schema: ::Hash[::String, untyped], + missing_fields: ::Set[::String], missing_types: ::Set[::String], - definition_conflicts: ::Set[SchemaElements::DeprecatedElement] - ) -> ::String? + definition_conflicts: ::Set[ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement], + missing_necessary_fields: ::Array[JSONSchemaWithMetadata::MissingNecessaryField] + ) -> void - def field_metadata_for: ( - ::String, - ::String, - missing_fields: ::Set[::String], - definition_conflicts: ::Set[SchemaElements::DeprecatedElement] - ) -> JSONSchemaFieldMetadata? + def with: ( + ?json_schema: ::Hash[::String, untyped], + ?missing_fields: ::Set[::String], + ?missing_types: ::Set[::String], + ?definition_conflicts: ::Set[ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement], + ?missing_necessary_fields: ::Array[JSONSchemaWithMetadata::MissingNecessaryField] + ) -> instance + end - def identify_missing_necessary_fields: ( - ::Hash[::String, untyped], - ::Hash[::String, ::String] - ) -> ::Array[MissingNecessaryField] + class JSONSchemaWithMetadata < JSONSchemaWithMetadataSupertype + def json_schema_version: () -> ::Integer - def identify_missing_necessary_fields_for_index_def: ( - indexableType, - Index, - JSONSchemaResolver, - ::Integer - ) -> ::Array[MissingNecessaryField] + class Merger + @field_metadata_by_type_and_field_name: ::Hash[::String, ::Hash[::String, JSONSchemaFieldMetadata]] + @renamed_types_by_old_name: ::Hash[::String, ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement] + @deleted_types_by_old_name: ::Hash[::String, ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement] + @renamed_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement]] + @deleted_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement]] + @state: ElasticGraph::SchemaDefinition::State + @derived_indexing_type_names: ::Set[::String] - class JSONSchemaResolver - @state: State - @old_type_name_by_current_name: ::Hash[::String, ::String] - @meta_by_old_type_and_name_in_index: ::Hash[::String, ::Hash[::String, ::Hash[::String, untyped]]] + attr_reader unused_deprecated_elements: ::Set[ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement] - def initialize: (State, ::Hash[::String, untyped], ::Hash[::String, ::String]) -> void - def necessary_path_missing?: (SchemaElements::FieldPath) -> bool + def initialize: (ElasticGraph::SchemaDefinition::Results) -> void + def merge_metadata_into: (::Hash[::String, untyped]) -> JSONSchemaWithMetadata private - def necessary_path_part_missing?: (::String, ::String) { (::Hash[::String, untyped]) -> void } -> bool + def determine_current_type_name: ( + ::String, + missing_types: ::Set[::String], + definition_conflicts: ::Set[ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement] + ) -> ::String? + + def field_metadata_for: ( + ::String, + ::String, + missing_fields: ::Set[::String], + definition_conflicts: ::Set[ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement] + ) -> JSONSchemaFieldMetadata? + + def identify_missing_necessary_fields: ( + ::Hash[::String, untyped], + ::Hash[::String, ::String] + ) -> ::Array[MissingNecessaryField] + + def identify_missing_necessary_fields_for_index_def: ( + ElasticGraph::SchemaDefinition::indexableType, + ElasticGraph::SchemaDefinition::Indexing::Index, + JSONSchemaResolver + ) -> ::Array[MissingNecessaryField] + + class JSONSchemaResolver + @state: ElasticGraph::SchemaDefinition::State + @old_type_name_by_current_name: ::Hash[::String, ::String] + @meta_by_old_type_and_name_in_index: ::Hash[::String, ::Hash[::String, ::Hash[::String, untyped]]] + + def initialize: (ElasticGraph::SchemaDefinition::State, ::Hash[::String, untyped], ::Hash[::String, ::String]) -> void + def necessary_path_missing?: (ElasticGraph::SchemaDefinition::SchemaElements::FieldPath) -> bool + + private + + def necessary_path_part_missing?: (::String, ::String) { (::Hash[::String, untyped]) -> void } -> bool + end end - end - class MissingNecessaryField - attr_reader field_type: ::String - attr_reader fully_qualified_path: ::String + class MissingNecessaryField + attr_reader field_type: ::String + attr_reader fully_qualified_path: ::String - def initialize: ( - field_type: ::String, - fully_qualified_path: ::String - ) -> void + def initialize: ( + field_type: ::String, + fully_qualified_path: ::String + ) -> void + + def with: ( + ?field_type: ::String, + ?fully_qualified_path: ::String + ) -> MissingNecessaryField + end end end end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/json_schema_pruner.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/json_schema_pruner.rbs index 8c5f323a2..898a7660d 100644 --- a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/json_schema_pruner.rbs +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/json_schema_pruner.rbs @@ -1,9 +1,14 @@ module ElasticGraph - module SchemaDefinition - class JSONSchemaPruner - def self.prune: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] - def self.referenced_type_names: (::Array[::String], ::Hash[::String, untyped]) -> ::Set[::String] - def self.collect_ref_names: (::Hash[::String, untyped]) -> ::Array[::String] + module JSONIngestion + module SchemaDefinition + class JSONSchemaPruner + def self.prune: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] + + private + + def self.referenced_type_names: (::Array[::String], ::Hash[::String, untyped]) -> ::Set[::String] + def self.collect_ref_names: (::Hash[::String, untyped]) -> ::Array[::String] + end end end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb index fa215a5f1..cd38c8eaa 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb @@ -77,7 +77,7 @@ def json_schema # @return [JSONSchemaFieldMetadata] additional ElasticGraph metadata to be stored in the JSON schema for this field. def json_schema_metadata - JSONSchemaFieldMetadata.new(type: type.name, name_in_index: name_in_index) + ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata.new(type: type.name, name_in_index: name_in_index) end # Builds a hash containing the mapping for the provided fields, normalizing it in the same way that the diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb index 160e3c854..0914fcb1e 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb @@ -114,7 +114,7 @@ def after_initialize end def json_schema_with_metadata_merger - @json_schema_with_metadata_merger ||= Indexing::JSONSchemaWithMetadata::Merger.new(self) + @json_schema_with_metadata_merger ||= ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::Merger.new(self) end def generate_datastore_config @@ -283,11 +283,13 @@ def build_public_json_schema .transform_values(&:to_json_schema) .compact + event_envelope = ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::EventEnvelope + { "$schema" => JSON_META_SCHEMA, JSON_SCHEMA_VERSION_KEY => json_schema_version, "$defs" => { - "ElasticGraphEventEnvelope" => Indexing::EventEnvelope.json_schema(root_document_type_names, json_schema_version) + "ElasticGraphEventEnvelope" => event_envelope.json_schema(root_document_type_names, json_schema_version) }.merge(definitions_by_name) } end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_artifact_manager.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_artifact_manager.rb index d64109ac2..b99438780 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_artifact_manager.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_artifact_manager.rb @@ -40,7 +40,7 @@ def initialize(schema_definition_results:, schema_artifacts_directory:, output:, @json_schemas_artifact = new_yaml_artifact( JSON_SCHEMAS_FILE, - JSONSchemaPruner.prune(schema_definition_results.current_public_json_schema), + ::ElasticGraph::JSONIngestion::SchemaDefinition::JSONSchemaPruner.prune(schema_definition_results.current_public_json_schema), extra_comment_lines: [ "This is the \"public\" JSON schema file and is intended to be provided to publishers so that", "they can perform code generation and event validation." @@ -193,7 +193,7 @@ def build_desired_versioned_json_schemas(current_public_json_schema) def report_json_schema_merge_errors(merged_results) json_schema_versions_by_missing_field = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[::String, ::Array[::Integer]] json_schema_versions_by_missing_type = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[::String, ::Array[::Integer]] - json_schema_versions_by_missing_necessary_field = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[Indexing::JSONSchemaWithMetadata::MissingNecessaryField, ::Array[::Integer]] + json_schema_versions_by_missing_necessary_field = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::MissingNecessaryField, ::Array[::Integer]] merged_results.each do |result| result.missing_fields.each do |field| diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field.rbs index 432e62034..2b21d08f7 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field.rbs @@ -50,7 +50,7 @@ module ElasticGraph @mapping: ::Hash[::String, untyped]? def mapping: () -> ::Hash[::String, untyped] def json_schema: () -> ::Hash[::String, untyped] - def json_schema_metadata: () -> JSONSchemaFieldMetadata + def json_schema_metadata: () -> ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata def self.normalized_mapping_hash_for: (::Array[Field]) -> ::Hash[::String, untyped] diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type.rbs index 5808a0709..c4f144bfd 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type.rbs @@ -4,7 +4,7 @@ module ElasticGraph interface _FieldType def to_mapping: () -> ::Hash[::String, untyped] def to_json_schema: () -> ::Hash[::String, untyped] - def json_schema_field_metadata_by_field_name: () -> ::Hash[::String, JSONSchemaFieldMetadata] + def json_schema_field_metadata_by_field_name: () -> ::Hash[::String, ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata] def format_field_json_schema_customizations: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/results.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/results.rbs index c35f971f2..41012f526 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/results.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/results.rbs @@ -10,9 +10,9 @@ module ElasticGraph include Support::_MemoizableDataClass def json_schema_version_setter_location: () -> ::Thread::Backtrace::Location? - def json_schema_field_metadata_by_type_and_field_name: () -> ::Hash[::String, ::Hash[::String, Indexing::JSONSchemaFieldMetadata]] + def json_schema_field_metadata_by_type_and_field_name: () -> ::Hash[::String, ::Hash[::String, ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata]] def current_public_json_schema: () -> ::Hash[::String, untyped] - def merge_field_metadata_into_json_schema: (::Hash[::String, untyped]) -> Indexing::JSONSchemaWithMetadata + def merge_field_metadata_into_json_schema: (::Hash[::String, untyped]) -> ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata def unused_deprecated_elements: () -> ::Set[SchemaElements::DeprecatedElement] def derived_indexing_type_names: () -> ::Set[::String] @@ -26,16 +26,16 @@ module ElasticGraph @field_path_resolver: SchemaElements::FieldPath::Resolver? @json_schema_indexing_field_types_by_name: ::Hash[::String, Indexing::_FieldType]? @derived_indexing_type_names: ::Set[::String]? - @json_schema_field_metadata_by_type_and_field_name: ::Hash[::String, ::Hash[::String, Indexing::JSONSchemaFieldMetadata]]? + @json_schema_field_metadata_by_type_and_field_name: ::Hash[::String, ::Hash[::String, ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata]]? @current_public_json_schema: ::Hash[::String, untyped]? @latest_versioned_json_schema: ::Hash[::String, untyped]? - @json_schema_with_metadata_merger: Indexing::JSONSchemaWithMetadata::Merger? + @json_schema_with_metadata_merger: ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::Merger? STATIC_SCRIPT_REPO: Scripting::FileSystemRepository private - def json_schema_with_metadata_merger: () -> Indexing::JSONSchemaWithMetadata::Merger + def json_schema_with_metadata_merger: () -> ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::Merger def generate_datastore_config: () -> ::Hash[::String, untyped] def build_dynamic_scripts: () -> ::Array[Scripting::Script] def build_runtime_metadata: () -> SchemaArtifacts::RuntimeMetadata::Schema diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_artifact_manager.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_artifact_manager.rbs index 0c6f3b97c..d529d9e07 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_artifact_manager.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_artifact_manager.rbs @@ -29,12 +29,12 @@ module ElasticGraph def notify_about_unused_type_name_overrides: () -> void def notify_about_unused_enum_value_overrides: () -> void def build_desired_versioned_json_schemas: (::Hash[::String, untyped]) -> ::Hash[::Integer, ::Hash[::String, untyped]] - def report_json_schema_merge_errors: (::Array[Indexing::JSONSchemaWithMetadata]) -> void + def report_json_schema_merge_errors: (::Array[::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata]) -> void def report_json_schema_merge_warnings: () -> void def format_deprecated_elements: (::Enumerable[SchemaElements::DeprecatedElement]) -> ::String def missing_field_error_for: (::String, ::Array[::Integer]) -> ::String def missing_type_error_for: (::String, ::Array[::Integer]) -> ::String - def missing_necessary_field_error_for: (Indexing::JSONSchemaWithMetadata::MissingNecessaryField, ::Array[::Integer]) -> ::String + def missing_necessary_field_error_for: (::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::MissingNecessaryField, ::Array[::Integer]) -> ::String def describe_json_schema_versions: (::Array[::Integer], ::String) -> ::String def old_versions: (::Array[::Integer]) -> ::String def files_noun_phrase: (::Array[::Integer]) -> ::String diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/json_schema_with_metadata_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/json_schema_with_metadata_spec.rb index 22fc317ca..f3cd503ac 100644 --- a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/json_schema_with_metadata_spec.rb +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/json_schema_with_metadata_spec.rb @@ -12,7 +12,7 @@ module ElasticGraph module SchemaDefinition module Indexing - ::RSpec.describe JSONSchemaWithMetadata do + ::RSpec.describe JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata do include_context "SchemaDefinitionHelpers" it "ignores derived indexed types that do not show up in the JSON schema" do @@ -1062,7 +1062,7 @@ def have_dumped_metadata(name_in_index, type) end def missing_necessary_field_of(field_type, fully_qualified_path) - JSONSchemaWithMetadata::MissingNecessaryField.new(field_type, fully_qualified_path) + described_class::MissingNecessaryField.new(field_type, fully_qualified_path) end end end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_field_metadata_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_field_metadata_spec.rb index 6c0aa3799..4e101b378 100644 --- a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_field_metadata_spec.rb +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_field_metadata_spec.rb @@ -6,6 +6,7 @@ # # frozen_string_literal: true +require "elastic_graph/json_ingestion/schema_definition/indexing/json_schema_field_metadata" require "elastic_graph/spec_support/schema_definition_helpers" module ElasticGraph @@ -144,7 +145,7 @@ def define_schema(&schema_definition) end def field_meta_of(type, name_in_index) - Indexing::JSONSchemaFieldMetadata.new(type: type, name_in_index: name_in_index) + JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata.new(type: type, name_in_index: name_in_index) end end end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_pruner_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_pruner_spec.rb index 66c140101..e9c01c9f2 100644 --- a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_pruner_spec.rb +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_pruner_spec.rb @@ -12,7 +12,7 @@ module ElasticGraph module SchemaDefinition - RSpec.describe JSONSchemaPruner do + RSpec.describe JSONIngestion::SchemaDefinition::JSONSchemaPruner do include_context "SchemaDefinitionHelpers" describe ".prune" do