Add new endpoints to configure data lifecycle on a data stream level. (…

…#94590)

With PR we introduce CRUD endpoints which update/delete the data lifecycle on the data stream level. When this is updated it will apply at the next DLM run to all the backing indices that are managed by DLM.

docs/build.gradle

@@ -93,6 +93,7 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
   systemProperty 'es.transport.cname_in_publish_address', 'true'
 
   requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
+  requiresFeature 'es.dlm_feature_flag_enabled', Version.fromString("8.8.0")
 
   // build the cluster with all plugins
   project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->
@@ -114,6 +115,9 @@ tasks.named("yamlRestTest").configure {
   doFirst {
     delete("${buildDir}/cluster/shared/repo")
   }
+  if (BuildParams.isSnapshotBuild() == false) {
+    systemProperty 'es.dlm_feature_flag_enabled', 'true'
+  }
 }
 
 // TODO: Remove the following when RCS feature is released

docs/changelog/94590.yaml

@@ -0,0 +1,5 @@
+pr: 94590
+summary: Add new endpoints to configure data lifecycle on a data stream level
+area: DLM
+type: feature
+issues: []

docs/reference/dlm/apis/delete-lifecycle.asciidoc

@@ -0,0 +1,84 @@
+[[dlm-delete-lifecycle]]
+=== Delete the lifecycle of a data stream
+++++
+<titleabbrev>Delete Data Stream Lifecycle</titleabbrev>
+++++
+
+experimental::[]
+
+Deletes the lifecycle from a set of data streams.
+
+[[dlm-delete-lifecycle-request]]
+==== {api-request-title}
+
+`DELETE _data_stream/<data-stream>/_lifecycle`
+
+[[dlm-delete-lifecycle-desc]]
+==== {api-description-title}
+
+Deletes the lifecycle from the specified data streams. If multiple data streams are provided but at least one of them
+does not exist, then the deletion of the lifecycle will fail for all of them and the API will respond with `404`.
+
+[[dlm-delete-lifecycle-path-params]]
+==== {api-path-parms-title}
+
+`<data-stream>`::
+(Required, string) Comma-separated list of data streams used to limit the request. Supports wildcards (`*`).
+To target all data streams use `*` or `_all`.
+
+
+[role="child_attributes"]
+[[delete-data-lifecycle-api-query-parms]]
+==== {api-query-parms-title}
+
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=ds-expand-wildcards]
++
+Defaults to `open`.
+
+[[dlm-delete-lifecycle-example]]
+==== {api-examples-title}
+
+////
+
+[source,console]
+--------------------------------------------------
+PUT /_index_template/my-template
+{
+  "index_patterns" : ["my-data-stream*"],
+  "priority" : 1,
+  "data_stream": {},
+  "template": {
+    "lifecycle" : {
+      "data_retention" : "7d"
+    }
+  }
+}
+
+PUT /_data_stream/my-data-stream
+--------------------------------------------------
+// TESTSETUP
+
+[source,console]
+--------------------------------------------------
+DELETE _data_stream/my-data-stream
+DELETE _index_template/my-template
+--------------------------------------------------
+// TEARDOWN
+
+////
+
+The following example deletes the lifecycle of `my-data-stream`:
+
+[source,console]
+--------------------------------------------------
+DELETE _data_stream/my-data-stream/_lifecycle
+--------------------------------------------------
+
+When the policy is successfully deleted from all selected data streams, you receive the following result:
+
+[source,console-result]
+--------------------------------------------------
+{
+  "acknowledged": true
+}
+--------------------------------------------------

docs/reference/dlm/apis/dlm-api.asciidoc

@@ -8,6 +8,13 @@ and to retrieve lifecycle information for backing indices.
 [[dlm-api-management-endpoint]]
 === Operation management APIs
 
+* <<dlm-put-lifecycle,Update data stream lifecycle>>
+* <<dlm-get-lifecycle,Get data stream lifecycle>>
+* <<dlm-delete-lifecycle,Delete data stream lifecycle>>
 * <<dlm-explain-lifecycle,Explain Lifecycle API>>
 
+include::put-lifecycle.asciidoc[]
+include::get-lifecycle.asciidoc[]
+include::delete-lifecycle.asciidoc[]
 include::explain-data-lifecycle.asciidoc[]
+

docs/reference/dlm/apis/get-lifecycle.asciidoc

@@ -0,0 +1,134 @@
+[[dlm-get-lifecycle]]
+=== Get the lifecycle of a data stream
+++++
+<titleabbrev>Get Data Stream Lifecycle</titleabbrev>
+++++
+
+experimental::[]
+
+Gets the lifecycle of a set of data streams.
+
+[[dlm-get-lifecycle-request]]
+==== {api-request-title}
+
+`GET _data_stream/<data-stream>/_lifecycle`
+
+[[dlm-get-lifecycle-desc]]
+==== {api-description-title}
+
+Gets the lifecycle of the specified data streams. If multiple data streams are requested but at least one of them
+does not exist, then the API will respond with `404` since at least one of the requested resources could not be retrieved.
+
+[[dlm-get-lifecycle-path-params]]
+==== {api-path-parms-title}
+
+`<data-stream>`::
+(Required, string) Comma-separated list of data streams used to limit the request. Supports wildcards (`*`).
+To target all data streams use `*` or `_all`.
+
+[role="child_attributes"]
+[[get-data-lifecycle-api-query-parms]]
+==== {api-query-parms-title}
+
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=ds-expand-wildcards]
++
+Defaults to `open`.
+
+`include_defaults`::
+(Optional, Boolean) If `true`, return all default settings in the response.
+Defaults to `false`.
+
+[role="child_attributes"]
+[[get-lifecycle-api-response-body]]
+==== {api-response-body-title}
+
+`data_streams`::
+(array of objects)
+Contains information about retrieved data stream lifecycles.
++
+.Properties of objects in `data_streams`
+[%collapsible%open]
+====
+`name`::
+(string)
+Name of the data stream.
+`lifecycle`::
+(object)
++
+.Properties of `lifecycle`
+[%collapsible%open]
+=====
+`data_retention`::
+(string)
+If defined, every document added to this data stream will be stored at least for this time frame. Any time after this
+duration the document could be deleted. When undefined, every document in this data stream will be stored indefinitely.
+
+`rollover`::
+(object)
+The conditions which will trigger the rollover of a backing index as configured by the cluster setting
+`cluster.dlm.default.rollover`. This property is an implementation detail and it will only be retrieved when the query
+param `include_defaults` is set to `true`. The contents of this field are subject to change.
+=====
+====
+
+[[dlm-get-lifecycle-example]]
+==== {api-examples-title}
+
+////
+
+[source,console]
+--------------------------------------------------
+PUT /_index_template/my-template
+{
+  "index_patterns" : ["my-data-stream*"],
+  "priority" : 1,
+  "data_stream": {},
+  "template": {
+    "lifecycle" : {
+      "data_retention" : "7d"
+    }
+  }
+}
+
+PUT /_data_stream/my-data-stream-1
+PUT /_data_stream/my-data-stream-2
+--------------------------------------------------
+// TESTSETUP
+
+[source,console]
+--------------------------------------------------
+DELETE _data_stream/my-data-stream*
+DELETE _index_template/my-template
+--------------------------------------------------
+// TEARDOWN
+
+////
+
+Let's retrieve the lifecycles:
+
+[source,console]
+--------------------------------------------------
+GET _data_stream/my-data-stream*/_lifecycle
+--------------------------------------------------
+
+The response will look like the following:
+
+[source,console-result]
+--------------------------------------------------
+{
+  "data_streams": [
+    {
+      "name": "my-data-stream-1",
+      "lifecycle": {
+        "data_retention": "7d"
+      }
+    },
+    {
+      "name": "my-data-stream-2",
+      "lifecycle": {
+        "data_retention": "7d"
+      }
+    }
+  ]
+}
+--------------------------------------------------

docs/reference/dlm/apis/put-lifecycle.asciidoc

@@ -0,0 +1,76 @@
+[[dlm-put-lifecycle]]
+=== Set the lifecycle of a data stream
+++++
+<titleabbrev>Put Data Stream Lifecycle</titleabbrev>
+++++
+
+experimental::[]
+
+Configures the data lifecycle for the targeted data streams.
+
+[[dlm-put-lifecycle-request]]
+==== {api-request-title}
+
+`PUT _data_stream/<data-stream>/_lifecycle`
+
+[[dlm-put-lifecycle-desc]]
+==== {api-description-title}
+
+Configures the data lifecycle for the targeted data streams. If multiple data streams are provided but at least one of them
+does not exist, then the update of the lifecycle will fail for all of them and the API will respond with `404`.
+
+[[dlm-put-lifecycle-path-params]]
+==== {api-path-parms-title}
+
+`<data-stream>`::
+(Required, string) Comma-separated list of data streams used to limit the request. Supports wildcards (`*`).
+To target all data streams use `*` or `_all`.
+
+[role="child_attributes"]
+[[put-data-lifecycle-api-query-parms]]
+==== {api-query-parms-title}
+
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=ds-expand-wildcards]
++
+Defaults to `open`.
+
+[[put-lifecycle-api-request-body]]
+==== {api-request-body-title}
+
+`lifecycle`::
+(Required, object)
++
+.Properties of `lifecycle`
+[%collapsible%open]
+====
+`data_retention`::
+(Optional, string)
+If defined, every document added to this data stream will be stored at least for this time frame. Any time after this
+duration the document could be deleted. When empty, every document in this data stream will be stored indefinitely.
+====
+
+[[dlm-put-lifecycle-example]]
+==== {api-examples-title}
+
+The following example sets the lifecycle of `my-data-stream`:
+
+[source,console]
+--------------------------------------------------
+PUT _data_stream/my-data-stream/_lifecycle
+{
+  "lifecycle": {
+    "data_retention": "7d"
+  }
+}
+--------------------------------------------------
+// TEST[setup:my_data_stream]
+// TEST[teardown:data_stream_cleanup]
+
+When the lifecycle is successfully updated in all data streams, you receive the following result:
+
+[source,console-result]
+--------------------------------------------------
+{
+  "acknowledged": true
+}
+--------------------------------------------------

docs/reference/rest-api/index.asciidoc

@@ -99,4 +99,3 @@ include::usage.asciidoc[]
 include::{xes-repo-dir}/rest-api/watcher.asciidoc[]
 include::defs.asciidoc[]
 include::{es-repo-dir}/dlm/apis/dlm-api.asciidoc[]
-

modules/data-streams/src/main/java/module-info.java

@@ -13,6 +13,6 @@
     requires org.apache.logging.log4j;
     requires org.apache.lucene.core;
 
-    exports org.elasticsearch.datastreams.action to org.elasticsearch.server;
+    exports org.elasticsearch.datastreams.action to org.elasticsearch.server, org.elasticsearch.dlm;
     exports org.elasticsearch.datastreams to org.elasticsearch.dlm;
 }

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsActionUtil.java

@@ -5,6 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
+
 package org.elasticsearch.datastreams.action;
 
 import org.elasticsearch.action.support.IndicesOptions;

modules/dlm/build.gradle

@@ -19,6 +19,7 @@ esplugin {
 }
 archivesBaseName = 'dlm'
 dependencies {
+  implementation project(path: ':modules:data-streams')
   testImplementation project(':modules:data-streams')
 }