From 111d19117b227d30079aa8f573bc2a0e7ad7d629 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 28 Jan 2025 09:26:55 -0600 Subject: [PATCH 1/4] Correcting the response format for ingest simulate --- output/openapi/elasticsearch-openapi.json | 47 +++++- output/schema/schema.json | 158 +++++++++++++++++- output/typescript/types.ts | 17 +- .../simulate/ingest/SimulateIngestResponse.ts | 50 +++++- 4 files changed, 267 insertions(+), 5 deletions(-) diff --git a/output/openapi/elasticsearch-openapi.json b/output/openapi/elasticsearch-openapi.json index b7d1a66880..ffb22277aa 100644 --- a/output/openapi/elasticsearch-openapi.json +++ b/output/openapi/elasticsearch-openapi.json @@ -88592,6 +88592,51 @@ "replace" ] }, + "simulate.ingest:SimulateDocumentResult": { + "type": "object", + "properties": { + "doc": { + "$ref": "#/components/schemas/simulate.ingest:DocumentSimulation" + }, + "error": { + "$ref": "#/components/schemas/_types:ErrorCause" + } + } + }, + "simulate.ingest:DocumentSimulation": { + "type": "object", + "properties": { + "_id": { + "$ref": "#/components/schemas/_types:Id" + }, + "_index": { + "$ref": "#/components/schemas/_types:IndexName" + }, + "_source": { + "description": "JSON body for the document.", + "type": "object", + "additionalProperties": { + "type": "object" + } + }, + "_version": { + "$ref": "#/components/schemas/_spec_utils:StringifiedVersionNumber" + }, + "executed_pipelines": { + "description": "A list of the names of the pipelines executed on this document.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "_id", + "_index", + "_source", + "executed_pipelines" + ] + }, "slm._types:SnapshotLifecycle": { "type": "object", "properties": { @@ -97474,7 +97519,7 @@ "docs": { "type": "array", "items": { - "$ref": "#/components/schemas/ingest._types:SimulateDocumentResult" + "$ref": "#/components/schemas/simulate.ingest:SimulateDocumentResult" } } }, diff --git a/output/schema/schema.json b/output/schema/schema.json index 685d65e04b..e08a4e821d 100644 --- a/output/schema/schema.json +++ b/output/schema/schema.json @@ -207600,6 +207600,127 @@ }, "specLocation": "shutdown/put_node/ShutdownPutNodeResponse.ts#L22-L24" }, + { + "kind": "interface", + "attachedBehaviors": [ + "AdditionalProperties" + ], + "behaviors": [ + { + "generics": [ + { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + }, + { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + ], + "meta": { + "description": "Additional metadata", + "fieldname": "metadata" + }, + "type": { + "name": "AdditionalProperties", + "namespace": "_spec_utils" + } + } + ], + "description": "The results of ingest simulation on a single document. The _source of the document contains\nthe results after running all pipelines listed in executed_pipelines on the document. The\nlist of executed pipelines is derived from the pipelines that would be executed if this\ndocument had been ingested into _index.", + "name": { + "name": "DocumentSimulation", + "namespace": "simulate.ingest" + }, + "properties": [ + { + "description": "Identifier for the document.", + "name": "_id", + "required": true, + "type": { + "kind": "instance_of", + "type": { + "name": "Id", + "namespace": "_types" + } + } + }, + { + "description": "Name of the index that the document would be indexed into if this were not a simulation.", + "name": "_index", + "required": true, + "type": { + "kind": "instance_of", + "type": { + "name": "IndexName", + "namespace": "_types" + } + } + }, + { + "description": "JSON body for the document.", + "name": "_source", + "required": true, + "type": { + "kind": "dictionary_of", + "key": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + }, + "singleKey": false, + "value": { + "kind": "user_defined_value" + } + } + }, + { + "description": "", + "name": "_version", + "required": false, + "type": { + "kind": "instance_of", + "generics": [ + { + "kind": "instance_of", + "type": { + "name": "VersionNumber", + "namespace": "_types" + } + } + ], + "type": { + "name": "Stringified", + "namespace": "_spec_utils" + } + } + }, + { + "description": "A list of the names of the pipelines executed on this document.", + "name": "executed_pipelines", + "required": true, + "type": { + "kind": "array_of", + "value": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + } + } + ], + "specLocation": "simulate/ingest/SimulateIngestResponse.ts#L41-L72" + }, { "kind": "request", "attachedBehaviors": [ @@ -207760,7 +207881,7 @@ "kind": "instance_of", "type": { "name": "SimulateDocumentResult", - "namespace": "ingest._types" + "namespace": "simulate.ingest" } } } @@ -207771,7 +207892,40 @@ "name": "Response", "namespace": "simulate.ingest" }, - "specLocation": "simulate/ingest/SimulateIngestResponse.ts#L22-L24" + "specLocation": "simulate/ingest/SimulateIngestResponse.ts#L27-L29" + }, + { + "kind": "interface", + "name": { + "name": "SimulateDocumentResult", + "namespace": "simulate.ingest" + }, + "properties": [ + { + "name": "doc", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "DocumentSimulation", + "namespace": "simulate.ingest" + } + } + }, + { + "description": "Any error resulting from simulatng ingest on this doc. This can be an error generated by\nexecuting a processor, or a mapping validation error when simulating indexing the resulting\ndoc.", + "name": "error", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "ErrorCause", + "namespace": "_types" + } + } + } + ], + "specLocation": "simulate/ingest/SimulateIngestResponse.ts#L31-L39" }, { "kind": "interface", diff --git a/output/typescript/types.ts b/output/typescript/types.ts index c5789a62aa..ba7ca7e8cd 100644 --- a/output/typescript/types.ts +++ b/output/typescript/types.ts @@ -19455,6 +19455,16 @@ export interface ShutdownPutNodeRequest extends RequestBase { export type ShutdownPutNodeResponse = AcknowledgedResponseBase +export interface SimulateIngestDocumentSimulationKeys { + _id: Id + _index: IndexName + _source: Record + _version?: SpecUtilsStringified + executed_pipelines: string[] +} +export type SimulateIngestDocumentSimulation = SimulateIngestDocumentSimulationKeys + & { [property: string]: string | Id | IndexName | Record | SpecUtilsStringified | string[] } + export interface SimulateIngestRequest extends RequestBase { index?: IndexName pipeline?: PipelineName @@ -19468,7 +19478,12 @@ export interface SimulateIngestRequest extends RequestBase { } export interface SimulateIngestResponse { - docs: IngestSimulateDocumentResult[] + docs: SimulateIngestSimulateDocumentResult[] +} + +export interface SimulateIngestSimulateDocumentResult { + doc?: SimulateIngestDocumentSimulation + error?: ErrorCause } export interface SlmConfiguration { diff --git a/specification/simulate/ingest/SimulateIngestResponse.ts b/specification/simulate/ingest/SimulateIngestResponse.ts index e1812124fb..7ce32dc887 100644 --- a/specification/simulate/ingest/SimulateIngestResponse.ts +++ b/specification/simulate/ingest/SimulateIngestResponse.ts @@ -17,8 +17,56 @@ * under the License. */ -import { SimulateDocumentResult } from '@ingest/_types/Simulation' +import { AdditionalProperties } from '@spec_utils/behaviors' +import { Dictionary } from '@spec_utils/Dictionary' +import { Stringified } from '@spec_utils/Stringified' +import { UserDefinedValue } from '@spec_utils/UserDefinedValue' +import { Id, IndexName, VersionNumber } from '@_types/common' +import { ErrorCause } from '@_types/Errors' export class Response { body: { docs: SimulateDocumentResult[] } } + +export class SimulateDocumentResult { + doc?: DocumentSimulation + /** + * Any error resulting from simulatng ingest on this doc. This can be an error generated by + * executing a processor, or a mapping validation error when simulating indexing the resulting + * doc. + */ + error?: ErrorCause +} + +/** + * The results of ingest simulation on a single document. The _source of the document contains + * the results after running all pipelines listed in executed_pipelines on the document. The + * list of executed pipelines is derived from the pipelines that would be executed if this + * document had been ingested into _index. + * + * @behavior_meta AdditionalProperties fieldname=metadata description="Additional metadata" + */ +export class DocumentSimulation + implements AdditionalProperties +{ + /** + * Identifier for the document. + */ + _id: Id + /** + * Name of the index that the document would be indexed into if this were not a simulation. + */ + _index: IndexName + /** + * JSON body for the document. + */ + _source: Dictionary + /** + * + */ + _version?: Stringified + /** + * A list of the names of the pipelines executed on this document. + */ + executed_pipelines: Array +} From e86ebc979aa0770abc353424e863ebb1018ef0af Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 31 Jan 2025 17:12:00 -0600 Subject: [PATCH 2/4] code review feedback --- output/schema/schema.json | 10 +++++----- output/typescript/types.ts | 12 ++++++------ .../simulate/ingest/SimulateIngestResponse.ts | 10 +++++----- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/output/schema/schema.json b/output/schema/schema.json index 5ef49239dd..8812bb5e0e 100644 --- a/output/schema/schema.json +++ b/output/schema/schema.json @@ -207587,7 +207587,7 @@ ], "description": "The results of ingest simulation on a single document. The _source of the document contains\nthe results after running all pipelines listed in executed_pipelines on the document. The\nlist of executed pipelines is derived from the pipelines that would be executed if this\ndocument had been ingested into _index.", "name": { - "name": "DocumentSimulation", + "name": "IngestDocumentSimulation", "namespace": "simulate.ingest" }, "properties": [ @@ -207637,7 +207637,7 @@ { "description": "", "name": "_version", - "required": false, + "required": true, "type": { "kind": "instance_of", "generics": [ @@ -207832,7 +207832,7 @@ "value": { "kind": "instance_of", "type": { - "name": "SimulateDocumentResult", + "name": "SimulateIngestDocumentResult", "namespace": "simulate.ingest" } } @@ -207849,7 +207849,7 @@ { "kind": "interface", "name": { - "name": "SimulateDocumentResult", + "name": "SimulateIngestDocumentResult", "namespace": "simulate.ingest" }, "properties": [ @@ -207859,7 +207859,7 @@ "type": { "kind": "instance_of", "type": { - "name": "DocumentSimulation", + "name": "IngestDocumentSimulation", "namespace": "simulate.ingest" } } diff --git a/output/typescript/types.ts b/output/typescript/types.ts index b5d46ae551..3290ce71f2 100644 --- a/output/typescript/types.ts +++ b/output/typescript/types.ts @@ -19448,14 +19448,14 @@ export interface ShutdownPutNodeRequest extends RequestBase { export type ShutdownPutNodeResponse = AcknowledgedResponseBase -export interface SimulateIngestDocumentSimulationKeys { +export interface SimulateIngestIngestDocumentSimulationKeys { _id: Id _index: IndexName _source: Record - _version?: SpecUtilsStringified + _version: SpecUtilsStringified executed_pipelines: string[] } -export type SimulateIngestDocumentSimulation = SimulateIngestDocumentSimulationKeys +export type SimulateIngestIngestDocumentSimulation = SimulateIngestIngestDocumentSimulationKeys & { [property: string]: string | Id | IndexName | Record | SpecUtilsStringified | string[] } export interface SimulateIngestRequest extends RequestBase { @@ -19471,11 +19471,11 @@ export interface SimulateIngestRequest extends RequestBase { } export interface SimulateIngestResponse { - docs: SimulateIngestSimulateDocumentResult[] + docs: SimulateIngestSimulateIngestDocumentResult[] } -export interface SimulateIngestSimulateDocumentResult { - doc?: SimulateIngestDocumentSimulation +export interface SimulateIngestSimulateIngestDocumentResult { + doc?: SimulateIngestIngestDocumentSimulation error?: ErrorCause } diff --git a/specification/simulate/ingest/SimulateIngestResponse.ts b/specification/simulate/ingest/SimulateIngestResponse.ts index 7ce32dc887..3a09697706 100644 --- a/specification/simulate/ingest/SimulateIngestResponse.ts +++ b/specification/simulate/ingest/SimulateIngestResponse.ts @@ -25,11 +25,11 @@ import { Id, IndexName, VersionNumber } from '@_types/common' import { ErrorCause } from '@_types/Errors' export class Response { - body: { docs: SimulateDocumentResult[] } + body: { docs: SimulateIngestDocumentResult[] } } -export class SimulateDocumentResult { - doc?: DocumentSimulation +export class SimulateIngestDocumentResult { + doc?: IngestDocumentSimulation /** * Any error resulting from simulatng ingest on this doc. This can be an error generated by * executing a processor, or a mapping validation error when simulating indexing the resulting @@ -46,7 +46,7 @@ export class SimulateDocumentResult { * * @behavior_meta AdditionalProperties fieldname=metadata description="Additional metadata" */ -export class DocumentSimulation +export class IngestDocumentSimulation implements AdditionalProperties { /** @@ -64,7 +64,7 @@ export class DocumentSimulation /** * */ - _version?: Stringified + _version: Stringified /** * A list of the names of the pipelines executed on this document. */ From e89fe60eb206775f74779edb1bc378acbedd99f4 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 31 Jan 2025 17:40:45 -0600 Subject: [PATCH 3/4] moving error to the correct place, and adding ignored_fields --- output/openapi/elasticsearch-openapi.json | 25 +++++-- output/schema/schema-serverless.json | 13 ++++ output/schema/schema.json | 67 +++++++++++++++---- output/typescript/types.ts | 7 +- .../simulate/ingest/SimulateIngestResponse.ts | 25 +++++-- 5 files changed, 108 insertions(+), 29 deletions(-) diff --git a/output/openapi/elasticsearch-openapi.json b/output/openapi/elasticsearch-openapi.json index 4f3912d2f1..4461d71016 100644 --- a/output/openapi/elasticsearch-openapi.json +++ b/output/openapi/elasticsearch-openapi.json @@ -88564,18 +88564,15 @@ "replace" ] }, - "simulate.ingest:SimulateDocumentResult": { + "simulate.ingest:SimulateIngestDocumentResult": { "type": "object", "properties": { "doc": { - "$ref": "#/components/schemas/simulate.ingest:DocumentSimulation" - }, - "error": { - "$ref": "#/components/schemas/_types:ErrorCause" + "$ref": "#/components/schemas/simulate.ingest:IngestDocumentSimulation" } } }, - "simulate.ingest:DocumentSimulation": { + "simulate.ingest:IngestDocumentSimulation": { "type": "object", "properties": { "_id": { @@ -88600,12 +88597,26 @@ "items": { "type": "string" } + }, + "ignored_fields": { + "description": "A list of the fields that would be ignored at the indexing step. For example, a field whose\nvalue is larger than the allowed limit would make it through all of the pipoelines, but\nwould not be indexed into Elasticsearch.", + "type": "array", + "items": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "error": { + "$ref": "#/components/schemas/_types:ErrorCause" } }, "required": [ "_id", "_index", "_source", + "_version", "executed_pipelines" ] }, @@ -97491,7 +97502,7 @@ "docs": { "type": "array", "items": { - "$ref": "#/components/schemas/simulate.ingest:SimulateDocumentResult" + "$ref": "#/components/schemas/simulate.ingest:SimulateIngestDocumentResult" } } }, diff --git a/output/schema/schema-serverless.json b/output/schema/schema-serverless.json index 165619dfb0..b3e8ae0276 100644 --- a/output/schema/schema-serverless.json +++ b/output/schema/schema-serverless.json @@ -105761,6 +105761,19 @@ }, "specLocation": "shutdown/get_node/ShutdownGetNodeResponse.ts#L40-L43" }, + { + "kind": "enum", + "members": [ + { + "name": "field" + } + ], + "name": { + "name": "IgnoredFieldKey", + "namespace": "simulate.ingest" + }, + "specLocation": "simulate/ingest/SimulateIngestResponse.ts#L80-L85" + }, { "kind": "type_alias", "name": { diff --git a/output/schema/schema.json b/output/schema/schema.json index 8812bb5e0e..913f802f15 100644 --- a/output/schema/schema.json +++ b/output/schema/schema.json @@ -207552,6 +207552,19 @@ }, "specLocation": "shutdown/put_node/ShutdownPutNodeResponse.ts#L22-L24" }, + { + "kind": "enum", + "members": [ + { + "name": "field" + } + ], + "name": { + "name": "IgnoredFieldKey", + "namespace": "simulate.ingest" + }, + "specLocation": "simulate/ingest/SimulateIngestResponse.ts#L80-L85" + }, { "kind": "interface", "attachedBehaviors": [ @@ -207669,9 +207682,47 @@ } } } + }, + { + "description": "A list of the fields that would be ignored at the indexing step. For example, a field whose\nvalue is larger than the allowed limit would make it through all of the pipoelines, but\nwould not be indexed into Elasticsearch.", + "name": "ignored_fields", + "required": false, + "type": { + "kind": "array_of", + "value": { + "kind": "dictionary_of", + "key": { + "kind": "instance_of", + "type": { + "name": "IgnoredFieldKey", + "namespace": "simulate.ingest" + } + }, + "singleKey": false, + "value": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + } + } + }, + { + "description": "Any error resulting from simulatng ingest on this doc. This can be an error generated by\nexecuting a processor, or a mapping validation error when simulating indexing the resulting\ndoc.", + "name": "error", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "ErrorCause", + "namespace": "_types" + } + } } ], - "specLocation": "simulate/ingest/SimulateIngestResponse.ts#L41-L72" + "specLocation": "simulate/ingest/SimulateIngestResponse.ts#L35-L78" }, { "kind": "request", @@ -207863,21 +207914,9 @@ "namespace": "simulate.ingest" } } - }, - { - "description": "Any error resulting from simulatng ingest on this doc. This can be an error generated by\nexecuting a processor, or a mapping validation error when simulating indexing the resulting\ndoc.", - "name": "error", - "required": false, - "type": { - "kind": "instance_of", - "type": { - "name": "ErrorCause", - "namespace": "_types" - } - } } ], - "specLocation": "simulate/ingest/SimulateIngestResponse.ts#L31-L39" + "specLocation": "simulate/ingest/SimulateIngestResponse.ts#L31-L33" }, { "kind": "interface", diff --git a/output/typescript/types.ts b/output/typescript/types.ts index 3290ce71f2..607df8229f 100644 --- a/output/typescript/types.ts +++ b/output/typescript/types.ts @@ -19448,15 +19448,19 @@ export interface ShutdownPutNodeRequest extends RequestBase { export type ShutdownPutNodeResponse = AcknowledgedResponseBase +export type SimulateIngestIgnoredFieldKey = 'field' + export interface SimulateIngestIngestDocumentSimulationKeys { _id: Id _index: IndexName _source: Record _version: SpecUtilsStringified executed_pipelines: string[] + ignored_fields?: Record[] + error?: ErrorCause } export type SimulateIngestIngestDocumentSimulation = SimulateIngestIngestDocumentSimulationKeys - & { [property: string]: string | Id | IndexName | Record | SpecUtilsStringified | string[] } + & { [property: string]: string | Id | IndexName | Record | SpecUtilsStringified | string[] | Record[] | ErrorCause } export interface SimulateIngestRequest extends RequestBase { index?: IndexName @@ -19476,7 +19480,6 @@ export interface SimulateIngestResponse { export interface SimulateIngestSimulateIngestDocumentResult { doc?: SimulateIngestIngestDocumentSimulation - error?: ErrorCause } export interface SlmConfiguration { diff --git a/specification/simulate/ingest/SimulateIngestResponse.ts b/specification/simulate/ingest/SimulateIngestResponse.ts index 3a09697706..fa7ea4ffe5 100644 --- a/specification/simulate/ingest/SimulateIngestResponse.ts +++ b/specification/simulate/ingest/SimulateIngestResponse.ts @@ -30,12 +30,6 @@ export class Response { export class SimulateIngestDocumentResult { doc?: IngestDocumentSimulation - /** - * Any error resulting from simulatng ingest on this doc. This can be an error generated by - * executing a processor, or a mapping validation error when simulating indexing the resulting - * doc. - */ - error?: ErrorCause } /** @@ -69,4 +63,23 @@ export class IngestDocumentSimulation * A list of the names of the pipelines executed on this document. */ executed_pipelines: Array + /** + * A list of the fields that would be ignored at the indexing step. For example, a field whose + * value is larger than the allowed limit would make it through all of the pipoelines, but + * would not be indexed into Elasticsearch. + */ + ignored_fields?: Array> + /** + * Any error resulting from simulatng ingest on this doc. This can be an error generated by + * executing a processor, or a mapping validation error when simulating indexing the resulting + * doc. + */ + error?: ErrorCause +} + +/** + * These are the names of the keys in a Dictionary in the ignored_fields Array. + */ +enum IgnoredFieldKey { + field } From 9b0367da4ba2ea1e6008c6af0cfde2c56e485ac0 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 4 Feb 2025 10:14:58 -0600 Subject: [PATCH 4/4] Update specification/simulate/ingest/SimulateIngestResponse.ts Co-authored-by: Laura Trotta <153528055+l-trotta@users.noreply.github.com> --- specification/simulate/ingest/SimulateIngestResponse.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/simulate/ingest/SimulateIngestResponse.ts b/specification/simulate/ingest/SimulateIngestResponse.ts index fa7ea4ffe5..f7250689dd 100644 --- a/specification/simulate/ingest/SimulateIngestResponse.ts +++ b/specification/simulate/ingest/SimulateIngestResponse.ts @@ -65,7 +65,7 @@ export class IngestDocumentSimulation executed_pipelines: Array /** * A list of the fields that would be ignored at the indexing step. For example, a field whose - * value is larger than the allowed limit would make it through all of the pipoelines, but + * value is larger than the allowed limit would make it through all of the pipelines, but * would not be indexed into Elasticsearch. */ ignored_fields?: Array>