diff --git a/docs/reference/ingest/apis/delete-pipeline.asciidoc b/docs/reference/ingest/apis/delete-pipeline.asciidoc new file mode 100644 index 0000000000000..468a5bf5d686f --- /dev/null +++ b/docs/reference/ingest/apis/delete-pipeline.asciidoc @@ -0,0 +1,79 @@ +[[delete-pipeline-api]] +=== Delete Pipeline API + +The delete pipeline API deletes pipelines by ID or wildcard match (`my-*`, `*`). + +////////////////////////// + +[source,js] +-------------------------------------------------- +PUT _ingest/pipeline/my-pipeline-id +{ + "description" : "describe pipeline", + "version" : 123, + "processors" : [ + { + "set" : { + "field": "foo", + "value": "bar" + } + } + ] +} +-------------------------------------------------- +// CONSOLE + +////////////////////////// + +[source,js] +-------------------------------------------------- +DELETE _ingest/pipeline/my-pipeline-id +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +////////////////////////// + +[source,js] +-------------------------------------------------- +{ +"acknowledged": true +} +-------------------------------------------------- +// TESTRESPONSE + +[source,js] +-------------------------------------------------- +PUT _ingest/pipeline/wild-one +{ + "description" : "first pipeline to be wildcard deleted", + "processors" : [ ] +} + +PUT _ingest/pipeline/wild-two +{ + "description" : "second pipeline to be wildcard deleted", + "processors" : [ ] +} +-------------------------------------------------- +// CONSOLE + +////////////////////////// + +[source,js] +-------------------------------------------------- +DELETE _ingest/pipeline/* +-------------------------------------------------- +// CONSOLE + +////////////////////////// + +[source,js] +-------------------------------------------------- +{ +"acknowledged": true +} +-------------------------------------------------- +// TESTRESPONSE + +////////////////////////// diff --git a/docs/reference/ingest/apis/get-pipeline.asciidoc b/docs/reference/ingest/apis/get-pipeline.asciidoc new file mode 100644 index 0000000000000..2f798f9fbd76b --- /dev/null +++ b/docs/reference/ingest/apis/get-pipeline.asciidoc @@ -0,0 +1,126 @@ +[[get-pipeline-api]] +=== Get Pipeline API + +The get pipeline API returns pipelines based on ID. This API always returns a local reference of the pipeline. + +////////////////////////// + +[source,js] +-------------------------------------------------- +PUT _ingest/pipeline/my-pipeline-id +{ + "description" : "describe pipeline", + "processors" : [ + { + "set" : { + "field": "foo", + "value": "bar" + } + } + ] +} +-------------------------------------------------- +// CONSOLE + +////////////////////////// + +[source,js] +-------------------------------------------------- +GET _ingest/pipeline/my-pipeline-id +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +Example response: + +[source,js] +-------------------------------------------------- +{ + "my-pipeline-id" : { + "description" : "describe pipeline", + "processors" : [ + { + "set" : { + "field" : "foo", + "value" : "bar" + } + } + ] + } +} +-------------------------------------------------- +// TESTRESPONSE + +For each returned pipeline, the source and the version are returned. +The version is useful for knowing which version of the pipeline the node has. +You can specify multiple IDs to return more than one pipeline. Wildcards are also supported. + +[float] +[[versioning-pipelines]] +==== Pipeline Versioning + +Pipelines can optionally add a `version` number, which can be any integer value, +in order to simplify pipeline management by external systems. The `version` +field is completely optional and it is meant solely for external management of +pipelines. To unset a `version`, simply replace the pipeline without specifying +one. + +[source,js] +-------------------------------------------------- +PUT _ingest/pipeline/my-pipeline-id +{ + "description" : "describe pipeline", + "version" : 123, + "processors" : [ + { + "set" : { + "field": "foo", + "value": "bar" + } + } + ] +} +-------------------------------------------------- +// CONSOLE + +To check for the `version`, you can +<> +using `filter_path` to limit the response to just the `version`: + +[source,js] +-------------------------------------------------- +GET /_ingest/pipeline/my-pipeline-id?filter_path=*.version +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +This should give a small response that makes it both easy and inexpensive to parse: + +[source,js] +-------------------------------------------------- +{ + "my-pipeline-id" : { + "version" : 123 + } +} +-------------------------------------------------- +// TESTRESPONSE + +////////////////////////// + +[source,js] +-------------------------------------------------- +DELETE /_ingest/pipeline/my-pipeline-id +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +[source,js] +-------------------------------------------------- +{ +"acknowledged": true +} +-------------------------------------------------- +// TESTRESPONSE + +////////////////////////// diff --git a/docs/reference/ingest/apis/put-pipeline.asciidoc b/docs/reference/ingest/apis/put-pipeline.asciidoc new file mode 100644 index 0000000000000..e27b4d1b50eed --- /dev/null +++ b/docs/reference/ingest/apis/put-pipeline.asciidoc @@ -0,0 +1,43 @@ +[[put-pipeline-api]] +=== Put Pipeline API + +The put pipeline API adds pipelines and updates existing pipelines in the cluster. + +[source,js] +-------------------------------------------------- +PUT _ingest/pipeline/my-pipeline-id +{ + "description" : "describe pipeline", + "processors" : [ + { + "set" : { + "field": "foo", + "value": "bar" + } + } + ] +} +-------------------------------------------------- +// CONSOLE + +////////////////////////// + +[source,js] +-------------------------------------------------- +DELETE /_ingest/pipeline/my-pipeline-id +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +[source,js] +-------------------------------------------------- +{ +"acknowledged": true +} +-------------------------------------------------- +// TESTRESPONSE + +////////////////////////// + +NOTE: The put pipeline API also instructs all ingest nodes to reload their in-memory representation of pipelines, so that + pipeline changes take effect immediately. diff --git a/docs/reference/ingest/apis/simulate-pipeline.asciidoc b/docs/reference/ingest/apis/simulate-pipeline.asciidoc new file mode 100644 index 0000000000000..d4f043e802159 --- /dev/null +++ b/docs/reference/ingest/apis/simulate-pipeline.asciidoc @@ -0,0 +1,258 @@ + +[[simulate-pipeline-api]] +=== Simulate Pipeline API + +The simulate pipeline API executes a specific pipeline against +the set of documents provided in the body of the request. + +You can either specify an existing pipeline to execute +against the provided documents, or supply a pipeline definition in +the body of the request. + +Here is the structure of a simulate request with a pipeline definition provided +in the body of the request: + +[source,js] +-------------------------------------------------- +POST _ingest/pipeline/_simulate +{ + "pipeline" : { + // pipeline definition here + }, + "docs" : [ + { "_source": {/** first document **/} }, + { "_source": {/** second document **/} }, + // ... + ] +} +-------------------------------------------------- +// NOTCONSOLE + +Here is the structure of a simulate request against an existing pipeline: + +[source,js] +-------------------------------------------------- +POST _ingest/pipeline/my-pipeline-id/_simulate +{ + "docs" : [ + { "_source": {/** first document **/} }, + { "_source": {/** second document **/} }, + // ... + ] +} +-------------------------------------------------- +// NOTCONSOLE + +Here is an example of a simulate request with a pipeline defined in the request +and its response: + +[source,js] +-------------------------------------------------- +POST _ingest/pipeline/_simulate +{ + "pipeline" : + { + "description": "_description", + "processors": [ + { + "set" : { + "field" : "field2", + "value" : "_value" + } + } + ] + }, + "docs": [ + { + "_index": "index", + "_type": "_doc", + "_id": "id", + "_source": { + "foo": "bar" + } + }, + { + "_index": "index", + "_type": "_doc", + "_id": "id", + "_source": { + "foo": "rab" + } + } + ] +} +-------------------------------------------------- +// CONSOLE + +Response: + +[source,js] +-------------------------------------------------- +{ + "docs": [ + { + "doc": { + "_id": "id", + "_index": "index", + "_type": "_doc", + "_source": { + "field2": "_value", + "foo": "bar" + }, + "_ingest": { + "timestamp": "2017-05-04T22:30:03.187Z" + } + } + }, + { + "doc": { + "_id": "id", + "_index": "index", + "_type": "_doc", + "_source": { + "field2": "_value", + "foo": "rab" + }, + "_ingest": { + "timestamp": "2017-05-04T22:30:03.188Z" + } + } + } + ] +} +-------------------------------------------------- +// TESTRESPONSE[s/"2017-05-04T22:30:03.187Z"/$body.docs.0.doc._ingest.timestamp/] +// TESTRESPONSE[s/"2017-05-04T22:30:03.188Z"/$body.docs.1.doc._ingest.timestamp/] + +[[ingest-verbose-param]] +==== Viewing Verbose Results +You can use the simulate pipeline API to see how each processor affects the ingest document +as it passes through the pipeline. To see the intermediate results of +each processor in the simulate request, you can add the `verbose` parameter +to the request. + +Here is an example of a verbose request and its response: + +[source,js] +-------------------------------------------------- +POST _ingest/pipeline/_simulate?verbose +{ + "pipeline" : + { + "description": "_description", + "processors": [ + { + "set" : { + "field" : "field2", + "value" : "_value2" + } + }, + { + "set" : { + "field" : "field3", + "value" : "_value3" + } + } + ] + }, + "docs": [ + { + "_index": "index", + "_type": "_doc", + "_id": "id", + "_source": { + "foo": "bar" + } + }, + { + "_index": "index", + "_type": "_doc", + "_id": "id", + "_source": { + "foo": "rab" + } + } + ] +} +-------------------------------------------------- +// CONSOLE + +Response: + +[source,js] +-------------------------------------------------- +{ + "docs": [ + { + "processor_results": [ + { + "doc": { + "_id": "id", + "_index": "index", + "_type": "_doc", + "_source": { + "field2": "_value2", + "foo": "bar" + }, + "_ingest": { + "timestamp": "2017-05-04T22:46:09.674Z" + } + } + }, + { + "doc": { + "_id": "id", + "_index": "index", + "_type": "_doc", + "_source": { + "field3": "_value3", + "field2": "_value2", + "foo": "bar" + }, + "_ingest": { + "timestamp": "2017-05-04T22:46:09.675Z" + } + } + } + ] + }, + { + "processor_results": [ + { + "doc": { + "_id": "id", + "_index": "index", + "_type": "_doc", + "_source": { + "field2": "_value2", + "foo": "rab" + }, + "_ingest": { + "timestamp": "2017-05-04T22:46:09.676Z" + } + } + }, + { + "doc": { + "_id": "id", + "_index": "index", + "_type": "_doc", + "_source": { + "field3": "_value3", + "field2": "_value2", + "foo": "rab" + }, + "_ingest": { + "timestamp": "2017-05-04T22:46:09.677Z" + } + } + } + ] + } + ] +} +-------------------------------------------------- +// TESTRESPONSE[s/"2017-05-04T22:46:09.674Z"/$body.docs.0.processor_results.0.doc._ingest.timestamp/] +// TESTRESPONSE[s/"2017-05-04T22:46:09.675Z"/$body.docs.0.processor_results.1.doc._ingest.timestamp/] +// TESTRESPONSE[s/"2017-05-04T22:46:09.676Z"/$body.docs.1.processor_results.0.doc._ingest.timestamp/] +// TESTRESPONSE[s/"2017-05-04T22:46:09.677Z"/$body.docs.1.processor_results.1.doc._ingest.timestamp/] diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc index 502210501f0c9..7f6bbb5302af9 100644 --- a/docs/reference/ingest/ingest-node.asciidoc +++ b/docs/reference/ingest/ingest-node.asciidoc @@ -30,417 +30,10 @@ The following ingest APIs are available for managing pipelines: * <> to delete a pipeline * <> to simulate a call to a pipeline -[[put-pipeline-api]] -=== Put Pipeline API - -The put pipeline API adds pipelines and updates existing pipelines in the cluster. - -[source,js] --------------------------------------------------- -PUT _ingest/pipeline/my-pipeline-id -{ - "description" : "describe pipeline", - "processors" : [ - { - "set" : { - "field": "foo", - "value": "bar" - } - } - ] -} --------------------------------------------------- -// CONSOLE - -NOTE: The put pipeline API also instructs all ingest nodes to reload their in-memory representation of pipelines, so that - pipeline changes take effect immediately. - -[[get-pipeline-api]] -=== Get Pipeline API - -The get pipeline API returns pipelines based on ID. This API always returns a local reference of the pipeline. - -[source,js] --------------------------------------------------- -GET _ingest/pipeline/my-pipeline-id --------------------------------------------------- -// CONSOLE -// TEST[continued] - -Example response: - -[source,js] --------------------------------------------------- -{ - "my-pipeline-id" : { - "description" : "describe pipeline", - "processors" : [ - { - "set" : { - "field" : "foo", - "value" : "bar" - } - } - ] - } -} --------------------------------------------------- -// TESTRESPONSE - -For each returned pipeline, the source and the version are returned. -The version is useful for knowing which version of the pipeline the node has. -You can specify multiple IDs to return more than one pipeline. Wildcards are also supported. - -[float] -[[versioning-pipelines]] -==== Pipeline Versioning - -Pipelines can optionally add a `version` number, which can be any integer value, -in order to simplify pipeline management by external systems. The `version` -field is completely optional and it is meant solely for external management of -pipelines. To unset a `version`, simply replace the pipeline without specifying -one. - -[source,js] --------------------------------------------------- -PUT _ingest/pipeline/my-pipeline-id -{ - "description" : "describe pipeline", - "version" : 123, - "processors" : [ - { - "set" : { - "field": "foo", - "value": "bar" - } - } - ] -} --------------------------------------------------- -// CONSOLE - -To check for the `version`, you can -<> -using `filter_path` to limit the response to just the `version`: - -[source,js] --------------------------------------------------- -GET /_ingest/pipeline/my-pipeline-id?filter_path=*.version --------------------------------------------------- -// CONSOLE -// TEST[continued] - -This should give a small response that makes it both easy and inexpensive to parse: - -[source,js] --------------------------------------------------- -{ - "my-pipeline-id" : { - "version" : 123 - } -} --------------------------------------------------- -// TESTRESPONSE - -[[delete-pipeline-api]] -=== Delete Pipeline API - -The delete pipeline API deletes pipelines by ID or wildcard match (`my-*`, `*`). - -[source,js] --------------------------------------------------- -DELETE _ingest/pipeline/my-pipeline-id --------------------------------------------------- -// CONSOLE -// TEST[continued] - -//// -Hidden setup for wildcard test: -[source,js] --------------------------------------------------- -PUT _ingest/pipeline/wild-one -{ - "description" : "first pipeline to be wildcard deleted", - "processors" : [ ] -} - -PUT _ingest/pipeline/wild-two -{ - "description" : "second pipeline to be wildcard deleted", - "processors" : [ ] -} - -DELETE _ingest/pipeline/* --------------------------------------------------- -// CONSOLE - -Hidden expected response: -[source,js] --------------------------------------------------- -{ -"acknowledged": true -} --------------------------------------------------- -// TESTRESPONSE -//// - -[[simulate-pipeline-api]] -=== Simulate Pipeline API - -The simulate pipeline API executes a specific pipeline against -the set of documents provided in the body of the request. - -You can either specify an existing pipeline to execute -against the provided documents, or supply a pipeline definition in -the body of the request. - -Here is the structure of a simulate request with a pipeline definition provided -in the body of the request: - -[source,js] --------------------------------------------------- -POST _ingest/pipeline/_simulate -{ - "pipeline" : { - // pipeline definition here - }, - "docs" : [ - { "_source": {/** first document **/} }, - { "_source": {/** second document **/} }, - // ... - ] -} --------------------------------------------------- -// NOTCONSOLE - -Here is the structure of a simulate request against an existing pipeline: - -[source,js] --------------------------------------------------- -POST _ingest/pipeline/my-pipeline-id/_simulate -{ - "docs" : [ - { "_source": {/** first document **/} }, - { "_source": {/** second document **/} }, - // ... - ] -} --------------------------------------------------- -// NOTCONSOLE - -Here is an example of a simulate request with a pipeline defined in the request -and its response: - -[source,js] --------------------------------------------------- -POST _ingest/pipeline/_simulate -{ - "pipeline" : - { - "description": "_description", - "processors": [ - { - "set" : { - "field" : "field2", - "value" : "_value" - } - } - ] - }, - "docs": [ - { - "_index": "index", - "_type": "_doc", - "_id": "id", - "_source": { - "foo": "bar" - } - }, - { - "_index": "index", - "_type": "_doc", - "_id": "id", - "_source": { - "foo": "rab" - } - } - ] -} --------------------------------------------------- -// CONSOLE - -Response: - -[source,js] --------------------------------------------------- -{ - "docs": [ - { - "doc": { - "_id": "id", - "_index": "index", - "_type": "_doc", - "_source": { - "field2": "_value", - "foo": "bar" - }, - "_ingest": { - "timestamp": "2017-05-04T22:30:03.187Z" - } - } - }, - { - "doc": { - "_id": "id", - "_index": "index", - "_type": "_doc", - "_source": { - "field2": "_value", - "foo": "rab" - }, - "_ingest": { - "timestamp": "2017-05-04T22:30:03.188Z" - } - } - } - ] -} --------------------------------------------------- -// TESTRESPONSE[s/"2017-05-04T22:30:03.187Z"/$body.docs.0.doc._ingest.timestamp/] -// TESTRESPONSE[s/"2017-05-04T22:30:03.188Z"/$body.docs.1.doc._ingest.timestamp/] - -[[ingest-verbose-param]] -==== Viewing Verbose Results -You can use the simulate pipeline API to see how each processor affects the ingest document -as it passes through the pipeline. To see the intermediate results of -each processor in the simulate request, you can add the `verbose` parameter -to the request. - -Here is an example of a verbose request and its response: - -[source,js] --------------------------------------------------- -POST _ingest/pipeline/_simulate?verbose -{ - "pipeline" : - { - "description": "_description", - "processors": [ - { - "set" : { - "field" : "field2", - "value" : "_value2" - } - }, - { - "set" : { - "field" : "field3", - "value" : "_value3" - } - } - ] - }, - "docs": [ - { - "_index": "index", - "_type": "_doc", - "_id": "id", - "_source": { - "foo": "bar" - } - }, - { - "_index": "index", - "_type": "_doc", - "_id": "id", - "_source": { - "foo": "rab" - } - } - ] -} --------------------------------------------------- -// CONSOLE - -Response: - -[source,js] --------------------------------------------------- -{ - "docs": [ - { - "processor_results": [ - { - "doc": { - "_id": "id", - "_index": "index", - "_type": "_doc", - "_source": { - "field2": "_value2", - "foo": "bar" - }, - "_ingest": { - "timestamp": "2017-05-04T22:46:09.674Z" - } - } - }, - { - "doc": { - "_id": "id", - "_index": "index", - "_type": "_doc", - "_source": { - "field3": "_value3", - "field2": "_value2", - "foo": "bar" - }, - "_ingest": { - "timestamp": "2017-05-04T22:46:09.675Z" - } - } - } - ] - }, - { - "processor_results": [ - { - "doc": { - "_id": "id", - "_index": "index", - "_type": "_doc", - "_source": { - "field2": "_value2", - "foo": "rab" - }, - "_ingest": { - "timestamp": "2017-05-04T22:46:09.676Z" - } - } - }, - { - "doc": { - "_id": "id", - "_index": "index", - "_type": "_doc", - "_source": { - "field3": "_value3", - "field2": "_value2", - "foo": "rab" - }, - "_ingest": { - "timestamp": "2017-05-04T22:46:09.677Z" - } - } - } - ] - } - ] -} --------------------------------------------------- -// TESTRESPONSE[s/"2017-05-04T22:46:09.674Z"/$body.docs.0.processor_results.0.doc._ingest.timestamp/] -// TESTRESPONSE[s/"2017-05-04T22:46:09.675Z"/$body.docs.0.processor_results.1.doc._ingest.timestamp/] -// TESTRESPONSE[s/"2017-05-04T22:46:09.676Z"/$body.docs.1.processor_results.0.doc._ingest.timestamp/] -// TESTRESPONSE[s/"2017-05-04T22:46:09.677Z"/$body.docs.1.processor_results.1.doc._ingest.timestamp/] +include::apis/put-pipeline.asciidoc[] +include::apis/get-pipeline.asciidoc[] +include::apis/delete-pipeline.asciidoc[] +include::apis/simulate-pipeline.asciidoc[] [[accessing-data-in-pipelines]] == Accessing Data in Pipelines