[7.9] GET data stream API returns additional information (#59128) (#5…

…9177)

* GET data stream API returns additional information (#59128)

This adds the data stream's index template, the configured ILM policy
(if any) and the health status of the data stream to the GET _data_stream
response.

Restoring a data stream from a snapshot could install a data stream that
doesn't match any composable templates. This also makes the `template`
field in the `GET _data_stream` response optional.

(cherry picked from commit 0d9c98a)
Signed-off-by: Andrei Dan <andrei.dan@elastic.co>

client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java

@@ -18,6 +18,8 @@
  */
 package org.elasticsearch.client.indices;
 
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -34,12 +36,21 @@ public final class DataStream {
     private final String timeStampField;
     private final List<String> indices;
     private final long generation;
+    ClusterHealthStatus dataStreamStatus;
+    @Nullable
+    String indexTemplate;
+    @Nullable
+    String ilmPolicyName;
 
-    public DataStream(String name, String timeStampField, List<String> indices, long generation) {
+    public DataStream(String name, String timeStampField, List<String> indices, long generation, ClusterHealthStatus dataStreamStatus,
+                      @Nullable String indexTemplate, @Nullable String ilmPolicyName) {
         this.name = name;
         this.timeStampField = timeStampField;
         this.indices = indices;
         this.generation = generation;
+        this.dataStreamStatus = dataStreamStatus;
+        this.indexTemplate = indexTemplate;
+        this.ilmPolicyName = ilmPolicyName;
     }
 
     public String getName() {
@@ -58,25 +69,49 @@ public long getGeneration() {
         return generation;
     }
 
+    public ClusterHealthStatus getDataStreamStatus() {
+        return dataStreamStatus;
+    }
+
+    public String getIndexTemplate() {
+        return indexTemplate;
+    }
+
+    public String getIlmPolicyName() {
+        return ilmPolicyName;
+    }
+
     public static final ParseField NAME_FIELD = new ParseField("name");
     public static final ParseField TIMESTAMP_FIELD_FIELD = new ParseField("timestamp_field");
     public static final ParseField INDICES_FIELD = new ParseField("indices");
     public static final ParseField GENERATION_FIELD = new ParseField("generation");
+    public static final ParseField STATUS_FIELD = new ParseField("status");
+    public static final ParseField INDEX_TEMPLATE_FIELD = new ParseField("template");
+    public static final ParseField ILM_POLICY_FIELD = new ParseField("ilm_policy");
 
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<DataStream, Void> PARSER = new ConstructingObjectParser<>("data_stream",
         args -> {
+            String dataStreamName = (String) args[0];
             String timeStampField = (String) ((Map<?, ?>) args[1]).get("name");
             List<String> indices =
                 ((List<Map<String, String>>) args[2]).stream().map(m -> m.get("index_name")).collect(Collectors.toList());
-            return new DataStream((String) args[0], timeStampField, indices, (Long) args[3]);
+            Long generation = (Long) args[3];
+            String statusStr = (String) args[4];
+            ClusterHealthStatus status = ClusterHealthStatus.fromString(statusStr);
+            String indexTemplate = (String) args[5];
+            String ilmPolicy = (String) args[6];
+            return new DataStream(dataStreamName, timeStampField, indices, generation, status, indexTemplate, ilmPolicy);
         });
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD);
         PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> p.map(), TIMESTAMP_FIELD_FIELD);
         PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> p.mapStrings(), INDICES_FIELD);
         PARSER.declareLong(ConstructingObjectParser.constructorArg(), GENERATION_FIELD);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), STATUS_FIELD);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), INDEX_TEMPLATE_FIELD);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), ILM_POLICY_FIELD);
     }
 
     public static DataStream fromXContent(XContentParser parser) throws IOException {
@@ -88,14 +123,17 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         DataStream that = (DataStream) o;
-        return name.equals(that.name) &&
+        return generation == that.generation &&
+            name.equals(that.name) &&
             timeStampField.equals(that.timeStampField) &&
             indices.equals(that.indices) &&
-            generation == that.generation;
+            dataStreamStatus == that.dataStreamStatus &&
+            Objects.equals(indexTemplate, that.indexTemplate) &&
+            Objects.equals(ilmPolicyName, that.ilmPolicyName);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, timeStampField, indices, generation);
+        return Objects.hash(name, timeStampField, indices, generation, dataStreamStatus, indexTemplate, ilmPolicyName);
     }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetDataStreamResponse.java

@@ -42,9 +42,11 @@ public List<DataStream> getDataStreams() {
 
     public static GetDataStreamResponse fromXContent(XContentParser parser) throws IOException {
         final List<DataStream> templates = new ArrayList<>();
-        for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_ARRAY; token = parser.nextToken()) {
-            if (token == XContentParser.Token.START_OBJECT) {
-                templates.add(DataStream.fromXContent(parser));
+        for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) {
+            if (token == XContentParser.Token.START_ARRAY) {
+                while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+                    templates.add(DataStream.fromXContent(parser));
+                }
             }
         }
         return new GetDataStreamResponse(templates);

client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java

@@ -20,7 +20,9 @@
 package org.elasticsearch.client.indices;
 
 import org.elasticsearch.action.admin.indices.datastream.GetDataStreamAction;
+import org.elasticsearch.action.admin.indices.datastream.GetDataStreamAction.Response.DataStreamInfo;
 import org.elasticsearch.client.AbstractResponseTestCase;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -48,17 +50,19 @@ private static List<Index> randomIndexInstances() {
         return indices;
     }
 
-    private static DataStream randomInstance() {
+    private static DataStreamInfo randomInstance() {
         List<Index> indices = randomIndexInstances();
         long generation = indices.size() + randomLongBetween(1, 128);
         String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
         indices.add(new Index(getDefaultBackingIndexName(dataStreamName, generation), UUIDs.randomBase64UUID(random())));
-        return new DataStream(dataStreamName, createTimestampField(randomAlphaOfLength(10)), indices, generation);
+        DataStream dataStream = new DataStream(dataStreamName, createTimestampField(randomAlphaOfLength(10)), indices, generation);
+        return new DataStreamInfo(dataStream, ClusterHealthStatus.YELLOW, randomAlphaOfLengthBetween(2, 10),
+            randomAlphaOfLengthBetween(2, 10));
     }
 
     @Override
     protected GetDataStreamAction.Response createServerTestInstance(XContentType xContentType) {
-        ArrayList<DataStream> dataStreams = new ArrayList<>();
+        ArrayList<DataStreamInfo> dataStreams = new ArrayList<>();
         int count = randomInt(10);
         for (int i = 0; i < count; i++) {
             dataStreams.add(randomInstance());
@@ -74,12 +78,12 @@ protected GetDataStreamResponse doParseToClientInstance(XContentParser parser) t
     @Override
     protected void assertInstances(GetDataStreamAction.Response serverTestInstance, GetDataStreamResponse clientInstance) {
         assertEquals(serverTestInstance.getDataStreams().size(), clientInstance.getDataStreams().size());
-        Iterator<DataStream> serverIt = serverTestInstance.getDataStreams().iterator();
+        Iterator<DataStreamInfo> serverIt = serverTestInstance.getDataStreams().iterator();
 
         Iterator<org.elasticsearch.client.indices.DataStream> clientIt = clientInstance.getDataStreams().iterator();
         while (serverIt.hasNext()) {
             org.elasticsearch.client.indices.DataStream client = clientIt.next();
-            DataStream server = serverIt.next();
+            DataStream server = serverIt.next().getDataStream();
             assertEquals(server.getName(), client.getName());
             assertEquals(server.getIndices().stream().map(Index::getName).collect(Collectors.toList()), client.getIndices());
             assertEquals(server.getTimeStampField().getName(), client.getTimeStampField());

rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml

@@ -13,6 +13,8 @@ setup:
               properties:
                 '@timestamp':
                   type: date
+            settings:
+              index.number_of_replicas: 0
           data_stream:
             timestamp_field: '@timestamp'
   - do:
@@ -49,16 +51,19 @@ setup:
   - do:
       indices.get_data_stream:
         name: "*"
-  - match: { 0.name: simple-data-stream1 }
-  - match: { 0.timestamp_field.name: '@timestamp' }
-  - match: { 0.generation: 1 }
-  - length: { 0.indices: 1 }
-  - match: { 0.indices.0.index_name: '.ds-simple-data-stream1-000001' }
-  - match: { 1.name: simple-data-stream2 }
-  - match: { 1.timestamp_field.name: '@timestamp2' }
-  - match: { 0.generation: 1 }
-  - length: { 1.indices: 1 }
-  - match: { 1.indices.0.index_name: '.ds-simple-data-stream2-000001' }
+  - match: { data_streams.0.name: simple-data-stream1 }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
+  - match: { data_streams.0.generation: 1 }
+  - length: { data_streams.0.indices: 1 }
+  - match: { data_streams.0.indices.0.index_name: '.ds-simple-data-stream1-000001' }
+  - match: { data_streams.0.status: 'GREEN' }
+  - match: { data_streams.0.template: 'my-template1' }
+  - match: { data_streams.1.name: simple-data-stream2 }
+  - match: { data_streams.1.timestamp_field.name: '@timestamp2' }
+  - match: { data_streams.0.generation: 1 }
+  - length: { data_streams.1.indices: 1 }
+  - match: { data_streams.1.indices.0.index_name: '.ds-simple-data-stream2-000001' }
+  - match: { data_streams.1.template: 'my-template2' }
 
   - do:
       index:
@@ -122,34 +127,34 @@ setup:
 
   - do:
       indices.get_data_stream: {}
-  - match: { 0.name: simple-data-stream1 }
-  - match: { 0.timestamp_field.name: '@timestamp' }
-  - match: { 0.timestamp_field.mapping: {type: date} }
-  - match: { 0.generation: 1 }
-  - match: { 1.name: simple-data-stream2 }
-  - match: { 1.timestamp_field.name: '@timestamp2' }
-  - match: { 1.timestamp_field.mapping: {type: date} }
-  - match: { 1.generation: 1 }
+  - match: { data_streams.0.name: simple-data-stream1 }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
+  - match: { data_streams.0.timestamp_field.mapping: {type: date} }
+  - match: { data_streams.0.generation: 1 }
+  - match: { data_streams.1.name: simple-data-stream2 }
+  - match: { data_streams.1.timestamp_field.name: '@timestamp2' }
+  - match: { data_streams.1.timestamp_field.mapping: {type: date} }
+  - match: { data_streams.1.generation: 1 }
 
   - do:
       indices.get_data_stream:
         name: simple-data-stream1
-  - match: { 0.name: simple-data-stream1 }
-  - match: { 0.timestamp_field.name: '@timestamp' }
-  - match: { 0.timestamp_field.mapping: {type: date} }
-  - match: { 0.generation: 1 }
+  - match: { data_streams.0.name: simple-data-stream1 }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
+  - match: { data_streams.0.timestamp_field.mapping: {type: date} }
+  - match: { data_streams.0.generation: 1 }
 
   - do:
       indices.get_data_stream:
         name: simple-data-stream*
-  - match: { 0.name: simple-data-stream1 }
-  - match: { 0.timestamp_field.name: '@timestamp' }
-  - match: { 0.timestamp_field.mapping: {type: date} }
-  - match: { 0.generation: 1 }
-  - match: { 1.name: simple-data-stream2 }
-  - match: { 1.timestamp_field.name: '@timestamp2' }
-  - match: { 1.timestamp_field.mapping: {type: date} }
-  - match: { 1.generation: 1 }
+  - match: { data_streams.0.name: simple-data-stream1 }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
+  - match: { data_streams.0.timestamp_field.mapping: {type: date} }
+  - match: { data_streams.0.generation: 1 }
+  - match: { data_streams.1.name: simple-data-stream2 }
+  - match: { data_streams.1.timestamp_field.name: '@timestamp2' }
+  - match: { data_streams.1.timestamp_field.mapping: {type: date} }
+  - match: { data_streams.1.generation: 1 }
 
   - do:
       indices.get_data_stream:
@@ -162,7 +167,7 @@ setup:
   - do:
       indices.get_data_stream:
         name: nonexistent*
-  - match: { $body: [] }
+  - match: { data_streams: [] }
 
   - do:
       indices.delete_data_stream:
@@ -202,11 +207,11 @@ setup:
 
   - do:
       indices.get_data_stream: {}
-  - match: { 0.name: simple-data-stream1 }
-  - match: { 0.timestamp_field.name: '@timestamp' }
-  - match: { 0.generation: 1 }
-  - length: { 0.indices: 1 }
-  - match: { 0.indices.0.index_name: '.ds-simple-data-stream1-000001' }
+  - match: { data_streams.0.name: simple-data-stream1 }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
+  - match: { data_streams.0.generation: 1 }
+  - length: { data_streams.0.indices: 1 }
+  - match: { data_streams.0.indices.0.index_name: '.ds-simple-data-stream1-000001' }
 
   - do:
       indices.delete_data_stream:

rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/30_auto_create_data_stream.yml

@@ -42,10 +42,10 @@
   - do:
       indices.get_data_stream:
         name: logs-foobar
-  - match: { 0.name: logs-foobar }
-  - match: { 0.timestamp_field.name: 'timestamp' }
-  - length: { 0.indices: 1 }
-  - match: { 0.indices.0.index_name: '.ds-logs-foobar-000001' }
+  - match: { data_streams.0.name: logs-foobar }
+  - match: { data_streams.0.timestamp_field.name: 'timestamp' }
+  - length: { data_streams.0.indices: 1 }
+  - match: { data_streams.0.indices.0.index_name: '.ds-logs-foobar-000001' }
 
   - do:
       indices.delete_data_stream:

rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml

@@ -57,11 +57,11 @@ setup:
   - do:
       indices.get_data_stream:
         name: "*"
-  - match: { 0.name: simple-data-stream }
-  - match: { 0.timestamp_field.name: '@timestamp' }
-  - match: { 0.generation: 2 }
-  - length: { 0.indices: 1 }
-  - match: { 0.indices.0.index_name: '.ds-simple-data-stream-000002' }
+  - match: { data_streams.0.name: simple-data-stream }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
+  - match: { data_streams.0.generation: 2 }
+  - length: { data_streams.0.indices: 1 }
+  - match: { data_streams.0.indices.0.index_name: '.ds-simple-data-stream-000002' }
 
   - do:
       indices.delete_data_stream:

rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml

@@ -45,12 +45,12 @@
   - do:
       indices.get_data_stream:
         name: "*"
-  - match: { 0.name: data-stream-for-rollover }
-  - match: { 0.timestamp_field.name: '@timestamp' }
-  - match: { 0.generation: 2 }
-  - length: { 0.indices: 2 }
-  - match: { 0.indices.0.index_name: '.ds-data-stream-for-rollover-000001' }
-  - match: { 0.indices.1.index_name: '.ds-data-stream-for-rollover-000002' }
+  - match: { data_streams.0.name: data-stream-for-rollover }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
+  - match: { data_streams.0.generation: 2 }
+  - length: { data_streams.0.indices: 2 }
+  - match: { data_streams.0.indices.0.index_name: '.ds-data-stream-for-rollover-000001' }
+  - match: { data_streams.0.indices.1.index_name: '.ds-data-stream-for-rollover-000002' }
 
   - do:
       indices.delete_data_stream:

server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java

@@ -30,16 +30,15 @@
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.admin.indices.template.delete.DeleteComposableIndexTemplateAction;
-import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.ingest.PutPipelineRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.support.replication.ReplicationRequest;
-import org.elasticsearch.cluster.metadata.DataStream;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -66,8 +65,8 @@
 import static org.elasticsearch.action.DocWriteRequest.OpType.CREATE;
 import static org.elasticsearch.action.DocWriteResponse.Result.CREATED;
 import static org.elasticsearch.action.DocWriteResponse.Result.UPDATED;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamServiceTests.generateMapping;
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.arrayWithSize;
@@ -262,11 +261,11 @@ public void testMixedAutoCreate() throws Exception {
         GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request("*");
         GetDataStreamAction.Response getDataStreamsResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet();
         assertThat(getDataStreamsResponse.getDataStreams(), hasSize(4));
-        getDataStreamsResponse.getDataStreams().sort(Comparator.comparing(DataStream::getName));
-        assertThat(getDataStreamsResponse.getDataStreams().get(0).getName(), equalTo("logs-foobar"));
-        assertThat(getDataStreamsResponse.getDataStreams().get(1).getName(), equalTo("logs-foobaz"));
-        assertThat(getDataStreamsResponse.getDataStreams().get(2).getName(), equalTo("logs-foobaz2"));
-        assertThat(getDataStreamsResponse.getDataStreams().get(3).getName(), equalTo("logs-foobaz3"));
+        getDataStreamsResponse.getDataStreams().sort(Comparator.comparing(dataStreamInfo -> dataStreamInfo.getDataStream().getName()));
+        assertThat(getDataStreamsResponse.getDataStreams().get(0).getDataStream().getName(), equalTo("logs-foobar"));
+        assertThat(getDataStreamsResponse.getDataStreams().get(1).getDataStream().getName(), equalTo("logs-foobaz"));
+        assertThat(getDataStreamsResponse.getDataStreams().get(2).getDataStream().getName(), equalTo("logs-foobaz2"));
+        assertThat(getDataStreamsResponse.getDataStreams().get(3).getDataStream().getName(), equalTo("logs-foobaz3"));
 
         GetIndexResponse getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices("logs-bar*")).actionGet();
         assertThat(getIndexResponse.getIndices(), arrayWithSize(4));