Keep track of timestamp_field mapping as part of a data stream (#58400)

Backporting #58096 to 7.x branch. Relates to #53100 * use mapping source direcly instead of using mapper service to extract the relevant mapping details * moved assertion to TimestampField class and added helper method for tests * Improved logic that inserts timestamp field mapping into an mapping. If the timestamp field path consisted out of object fields and if the final mapping did not contain the parent field then an error occurred, because the prior logic assumed that the object field existed.
elastic · Jun 22, 2020 · 7dda993 · 7dda993
1 parent 765f1b5
commit 7dda993
Show file tree

Hide file tree

Showing 35 changed files with 514 additions and 125 deletions.
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java
@@ -20,8 +20,6 @@
 
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
-import org.elasticsearch.common.xcontent.ToXContentObject;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -30,12 +28,12 @@
 import java.util.Objects;
 import java.util.stream.Collectors;
 
-public final class DataStream implements ToXContentObject {
+public final class DataStream {
 
     private final String name;
     private final String timeStampField;
     private final List<String> indices;
-    private long generation;
+    private final long generation;
 
     public DataStream(String name, String timeStampField, List<String> indices, long generation) {
         this.name = name;
@@ -68,14 +66,15 @@ public long getGeneration() {
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<DataStream, Void> PARSER = new ConstructingObjectParser<>("data_stream",
         args -> {
+            String timeStampField = (String) ((Map<?, ?>) args[1]).get("name");
             List<String> indices =
                 ((List<Map<String, String>>) args[2]).stream().map(m -> m.get("index_name")).collect(Collectors.toList());
-            return new DataStream((String) args[0], (String) args[1], indices, (Long) args[3]);
+            return new DataStream((String) args[0], timeStampField, indices, (Long) args[3]);
         });
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD);
-        PARSER.declareString(ConstructingObjectParser.constructorArg(), TIMESTAMP_FIELD_FIELD);
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> p.map(), TIMESTAMP_FIELD_FIELD);
         PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> p.mapStrings(), INDICES_FIELD);
         PARSER.declareLong(ConstructingObjectParser.constructorArg(), GENERATION_FIELD);
     }
@@ -84,17 +83,6 @@ public static DataStream fromXContent(XContentParser parser) throws IOException
         return PARSER.parse(parser, null);
     }
 
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        builder.field(NAME_FIELD.getPreferredName(), name);
-        builder.field(TIMESTAMP_FIELD_FIELD.getPreferredName(), timeStampField);
-        builder.field(INDICES_FIELD.getPreferredName(), indices);
-        builder.field(GENERATION_FIELD.getPreferredName(), generation);
-        builder.endObject();
-        return builder;
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;

diff --git a/...high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java b/...high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java
@@ -34,6 +34,7 @@
 import java.util.Locale;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField;
 import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName;
 
 public class GetDataStreamResponseTests extends AbstractResponseTestCase<GetDataStreamAction.Response, GetDataStreamResponse> {
@@ -52,12 +53,7 @@ private static DataStream randomInstance() {
         long generation = indices.size() + randomLongBetween(1, 128);
         String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
         indices.add(new Index(getDefaultBackingIndexName(dataStreamName, generation), UUIDs.randomBase64UUID(random())));
-        return new DataStream(dataStreamName, randomAlphaOfLength(10), indices, generation);
-    }
-
-    private static GetDataStreamResponse fromXContent(XContentParser parser) throws IOException {
-        parser.nextToken();
-        return GetDataStreamResponse.fromXContent(parser);
+        return new DataStream(dataStreamName, createTimestampField(randomAlphaOfLength(10)), indices, generation);
     }
 
     @Override
@@ -86,7 +82,7 @@ protected void assertInstances(GetDataStreamAction.Response serverTestInstance,
             DataStream server = serverIt.next();
             assertEquals(server.getName(), client.getName());
             assertEquals(server.getIndices().stream().map(Index::getName).collect(Collectors.toList()), client.getIndices());
-            assertEquals(server.getTimeStampField(), client.getTimeStampField());
+            assertEquals(server.getTimeStampField().getName(), client.getTimeStampField());
             assertEquals(server.getGeneration(), client.getGeneration());
         }
     }

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml
@@ -50,12 +50,12 @@ setup:
       indices.get_data_stream:
         name: "*"
   - match: { 0.name: simple-data-stream1 }
-  - match: { 0.timestamp_field: '@timestamp' }
+  - match: { 0.timestamp_field.name: '@timestamp' }
   - match: { 0.generation: 1 }
   - length: { 0.indices: 1 }
   - match: { 0.indices.0.index_name: '.ds-simple-data-stream1-000001' }
   - match: { 1.name: simple-data-stream2 }
-  - match: { 1.timestamp_field: '@timestamp2' }
+  - match: { 1.timestamp_field.name: '@timestamp2' }
   - match: { 0.generation: 1 }
   - length: { 1.indices: 1 }
   - match: { 1.indices.0.index_name: '.ds-simple-data-stream2-000001' }
@@ -121,27 +121,32 @@ setup:
   - do:
       indices.get_data_stream: {}
   - match: { 0.name: simple-data-stream1 }
-  - match: { 0.timestamp_field: '@timestamp' }
+  - match: { 0.timestamp_field.name: '@timestamp' }
+  - match: { 0.timestamp_field.mapping: {type: date} }
   - match: { 0.generation: 1 }
   - match: { 1.name: simple-data-stream2 }
-  - match: { 1.timestamp_field: '@timestamp2' }
+  - match: { 1.timestamp_field.name: '@timestamp2' }
+  - match: { 1.timestamp_field.mapping: {type: date} }
   - match: { 1.generation: 1 }
 
   - do:
       indices.get_data_stream:
         name: simple-data-stream1
   - match: { 0.name: simple-data-stream1 }
-  - match: { 0.timestamp_field: '@timestamp' }
+  - match: { 0.timestamp_field.name: '@timestamp' }
+  - match: { 0.timestamp_field.mapping: {type: date} }
   - match: { 0.generation: 1 }
 
   - do:
       indices.get_data_stream:
         name: simple-data-stream*
   - match: { 0.name: simple-data-stream1 }
-  - match: { 0.timestamp_field: '@timestamp' }
+  - match: { 0.timestamp_field.name: '@timestamp' }
+  - match: { 0.timestamp_field.mapping: {type: date} }
   - match: { 0.generation: 1 }
   - match: { 1.name: simple-data-stream2 }
-  - match: { 1.timestamp_field: '@timestamp2' }
+  - match: { 1.timestamp_field.name: '@timestamp2' }
+  - match: { 1.timestamp_field.mapping: {type: date} }
   - match: { 1.generation: 1 }
 
   - do:
@@ -196,7 +201,7 @@ setup:
   - do:
       indices.get_data_stream: {}
   - match: { 0.name: simple-data-stream1 }
-  - match: { 0.timestamp_field: '@timestamp' }
+  - match: { 0.timestamp_field.name: '@timestamp' }
   - match: { 0.generation: 1 }
   - length: { 0.indices: 1 }
   - match: { 0.indices.0.index_name: '.ds-simple-data-stream1-000001' }

diff --git a/.../src/main/resources/rest-api-spec/test/indices.data_stream/30_auto_create_data_stream.yml b/.../src/main/resources/rest-api-spec/test/indices.data_stream/30_auto_create_data_stream.yml
@@ -41,7 +41,7 @@
       indices.get_data_stream:
         name: logs-foobar
   - match: { 0.name: logs-foobar }
-  - match: { 0.timestamp_field: 'timestamp' }
+  - match: { 0.timestamp_field.name: 'timestamp' }
   - length: { 0.indices: 1 }
   - match: { 0.indices.0.index_name: '.ds-logs-foobar-000001' }
 

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml
@@ -58,7 +58,7 @@ setup:
       indices.get_data_stream:
         name: "*"
   - match: { 0.name: simple-data-stream }
-  - match: { 0.timestamp_field: '@timestamp' }
+  - match: { 0.timestamp_field.name: '@timestamp' }
   - match: { 0.generation: 2 }
   - length: { 0.indices: 1 }
   - match: { 0.indices.0.index_name: '.ds-simple-data-stream-000002' }

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml
@@ -46,7 +46,7 @@
       indices.get_data_stream:
         name: "*"
   - match: { 0.name: data-stream-for-rollover }
-  - match: { 0.timestamp_field: '@timestamp' }
+  - match: { 0.timestamp_field.name: '@timestamp' }
   - match: { 0.generation: 2 }
   - length: { 0.indices: 2 }
   - match: { 0.indices.0.index_name: '.ds-data-stream-for-rollover-000001' }