Support non-keyword dimensions as routing fields in TSDB (#105501)

* Fix test failure https://gradle-enterprise.elastic.co/s/icg66i6mwnjoi * Fix test failure https://gradle-enterprise.elastic.co/s/icg66i6mwnjoi * Nest pass-through objects within objects * Support numeric fields as pass-through dimensions * Update docs/changelog/105073.yaml * fix tests * fix tests * refactor dimension property * fix yaml * add numeric to routing builder * fix violation * fix yaml versions * handle all numeric values in dimension fields * use shardId in TSDB doc id * spotless fix * test fixes * index version guard * test fix * revert shardId changes * add routing id field mapper * Update docs/changelog/105501.yaml * fix shrunk test * use SortedDocValuesField * fix tests * delete unused yaml * submit test fixes * sync * restrict id override to latest version * update painless path * test fix * check for invalid id * fix painless tests * restore bwc routing changes * add unittest * refactor passthrough test * address comments * remove redundant variable * remove assert * fix id test * fix painless * skip storing routing field * skip generating the id when available * skip assert on routing in TranslogWriter * skip id mismatch assert * changelog update * revert redundant changes * remove supportsDimension * spotless * rename var for forcing dimension * small refactor * small refactor
elastic · Mar 21, 2024 · 19cffde · 19cffde
1 parent c4e9646
commit 19cffde
Show file tree

Hide file tree

Showing 22 changed files with 178 additions and 184 deletions.
diff --git a/docs/changelog/105501.yaml b/docs/changelog/105501.yaml
@@ -0,0 +1,5 @@
+pr: 105501
+summary: Support non-keyword dimensions as routing fields in TSDB
+area: TSDB
+type: enhancement
+issues: []
diff --git a/...ta-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/...ta-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java
@@ -260,7 +260,7 @@ public void testInvalidTsdbTemplatesNoTimeSeriesDimensionAttribute() throws Exce
             assertThat(
                 e.getCause().getCause().getMessage(),
                 equalTo(
-                    "All fields that match routing_path must be keywords with [time_series_dimension: true] "
+                    "All fields that match routing_path must be configured with [time_series_dimension: true] "
                         + "or flattened fields with a list of dimensions in [time_series_dimensions] and "
                         + "without the [script] parameter. [metricset] was not a dimension."
                 )
@@ -289,7 +289,7 @@ public void testInvalidTsdbTemplatesNoTimeSeriesDimensionAttribute() throws Exce
         }
     }
 
-    public void testInvalidTsdbTemplatesNoKeywordFieldType() throws Exception {
+    public void testTsdbTemplatesNoKeywordFieldType() throws Exception {
         var mappingTemplate = """
             {
               "_doc":{
@@ -315,18 +315,7 @@ public void testInvalidTsdbTemplatesNoKeywordFieldType() throws Exception {
                 .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
                 .build()
         );
-        Exception e = expectThrows(
-            IllegalArgumentException.class,
-            () -> client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet()
-        );
-        assertThat(
-            e.getCause().getCause().getMessage(),
-            equalTo(
-                "All fields that match routing_path must be keywords with [time_series_dimension: true] "
-                    + "or flattened fields with a list of dimensions in [time_series_dimensions] and "
-                    + "without the [script] parameter. [metricset] was [long]."
-            )
-        );
+        client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet();
     }
 
     public void testInvalidTsdbTemplatesMissingSettings() throws Exception {

diff --git a/...src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/...src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java
@@ -53,6 +53,17 @@ public class TSDBPassthroughIndexingIT extends ESSingleNodeTestCase {
     public static final String MAPPING_TEMPLATE = """
         {
           "_doc":{
+            "dynamic_templates": [
+                {
+                  "strings_as_ip": {
+                    "match_mapping_type": "string",
+                    "match": "*ip",
+                    "mapping": {
+                      "type": "ip"
+                    }
+                  }
+                }
+            ],
             "properties": {
               "@timestamp" : {
                 "type": "date"
@@ -87,6 +98,8 @@ public class TSDBPassthroughIndexingIT extends ESSingleNodeTestCase {
             "@timestamp": "$time",
             "attributes": {
                 "metricset": "pod",
+                "number.long": $number1,
+                "number.double": $number2,
                 "pod": {
                     "name": "$name",
                     "uid": "$uid",
@@ -102,6 +115,15 @@ public class TSDBPassthroughIndexingIT extends ESSingleNodeTestCase {
         }
         """;
 
+    private static String getRandomDoc(Instant time) {
+        return DOC.replace("$time", formatInstant(time))
+            .replace("$uid", randomUUID())
+            .replace("$name", randomAlphaOfLength(4))
+            .replace("$number1", Long.toString(randomLong()))
+            .replace("$number2", Double.toString(randomDouble()))
+            .replace("$ip", InetAddresses.toAddrString(randomIp(randomBoolean())));
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return List.of(DataStreamsPlugin.class, InternalSettingsPlugin.class);
@@ -137,13 +159,7 @@ public void testIndexingGettingAndSearching() throws Exception {
         Instant time = Instant.now();
         for (int i = 0; i < indexingIters; i++) {
             var indexRequest = new IndexRequest("k8s").opType(DocWriteRequest.OpType.CREATE);
-            indexRequest.source(
-                DOC.replace("$time", formatInstant(time))
-                    .replace("$uid", randomUUID())
-                    .replace("$name", randomAlphaOfLength(4))
-                    .replace("$ip", InetAddresses.toAddrString(randomIp(randomBoolean()))),
-                XContentType.JSON
-            );
+            indexRequest.source(getRandomDoc(time), XContentType.JSON);
             var indexResponse = client().index(indexRequest).actionGet();
             index = indexResponse.getIndex();
             String id = indexResponse.getId();
@@ -176,14 +192,24 @@ public void testIndexingGettingAndSearching() throws Exception {
         );
         @SuppressWarnings("unchecked")
         var attributes = (Map<String, Map<?, ?>>) ObjectPath.eval("properties.attributes.properties", mapping);
-        assertMap(attributes.get("pod.ip"), matchesMap().entry("type", "keyword").entry("time_series_dimension", true));
+        assertMap(attributes.get("number.long"), matchesMap().entry("type", "long").entry("time_series_dimension", true));
+        assertMap(attributes.get("number.double"), matchesMap().entry("type", "float").entry("time_series_dimension", true));
+        assertMap(attributes.get("pod.ip"), matchesMap().entry("type", "ip").entry("time_series_dimension", true));
         assertMap(attributes.get("pod.uid"), matchesMap().entry("type", "keyword").entry("time_series_dimension", true));
         assertMap(attributes.get("pod.name"), matchesMap().entry("type", "keyword").entry("time_series_dimension", true));
         // alias field mappers:
         assertMap(
             ObjectPath.eval("properties.metricset", mapping),
             matchesMap().entry("type", "alias").entry("path", "attributes.metricset")
         );
+        assertMap(
+            ObjectPath.eval("properties.number.properties.long", mapping),
+            matchesMap().entry("type", "alias").entry("path", "attributes.number.long")
+        );
+        assertMap(
+            ObjectPath.eval("properties.number.properties.double", mapping),
+            matchesMap().entry("type", "alias").entry("path", "attributes.number.double")
+        );
         assertMap(
             ObjectPath.eval("properties.pod.properties", mapping),
             matchesMap().extraOk().entry("name", matchesMap().entry("type", "alias").entry("path", "attributes.pod.name"))
@@ -220,13 +246,7 @@ public void testIndexingGettingAndSearchingShrunkIndex() throws Exception {
         var bulkRequest = new BulkRequest(dataStreamName);
         for (int i = 0; i < numBulkItems; i++) {
             var indexRequest = new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.CREATE);
-            indexRequest.source(
-                DOC.replace("$time", formatInstant(time))
-                    .replace("$uid", randomUUID())
-                    .replace("$name", randomAlphaOfLength(4))
-                    .replace("$ip", InetAddresses.toAddrString(randomIp(randomBoolean()))),
-                XContentType.JSON
-            );
+            indexRequest.source(getRandomDoc(time), XContentType.JSON);
             bulkRequest.add(indexRequest);
             time = time.plusMillis(1);
         }

diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml
@@ -231,13 +231,13 @@ dynamic templates:
         refresh: true
         body:
           - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
-          - '{ "@timestamp": "2023-09-01T13:03:08.138Z", "data": "10", "attributes.dim1": "A", "attributes.dim2": "1", "attributes.another.dim1": "C", "attributes.another.dim2": "10" }'
+          - '{ "@timestamp": "2023-09-01T13:03:08.138Z", "data": "10", "attributes.dim1": "A", "attributes.dim2": "1", "attributes.another.dim1": "C", "attributes.another.dim2": "10.5" }'
           - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
-          - '{ "@timestamp": "2023-09-01T13:03:09.138Z", "data": "20", "attributes.dim1": "A", "attributes.dim2": "1", "attributes.another.dim1": "C", "attributes.another.dim2": "10" }'
+          - '{ "@timestamp": "2023-09-01T13:03:09.138Z", "data": "20", "attributes.dim1": "A", "attributes.dim2": "1", "attributes.another.dim1": "C", "attributes.another.dim2": "10.5" }'
           - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
-          - '{ "@timestamp": "2023-09-01T13:03:10.138Z", "data": "30", "attributes.dim1": "B", "attributes.dim2": "2", "attributes.another.dim1": "D", "attributes.another.dim2": "20" }'
+          - '{ "@timestamp": "2023-09-01T13:03:10.138Z", "data": "30", "attributes.dim1": "B", "attributes.dim2": "2", "attributes.another.dim1": "D", "attributes.another.dim2": "20.5" }'
           - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
-          - '{ "@timestamp": "2023-09-01T13:03:10.238Z", "data": "40", "attributes.dim1": "B", "attributes.dim2": "2", "attributes.another.dim1": "D", "attributes.another.dim2": "20" }'
+          - '{ "@timestamp": "2023-09-01T13:03:10.238Z", "data": "40", "attributes.dim1": "B", "attributes.dim2": "2", "attributes.another.dim1": "D", "attributes.another.dim2": "20.5" }'
 
   - do:
       search:
@@ -263,7 +263,7 @@ dynamic templates:
                     field: _tsid
 
   - length: { aggregations.filterA.tsids.buckets: 1 }
-  - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiK8yYWLhfZ18WLDvTuBX1YJX1Ll7UMNJqYNES5Eg" }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" }
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
   - do:
@@ -282,7 +282,7 @@ dynamic templates:
                     field: _tsid
 
   - length: { aggregations.filterA.tsids.buckets: 1 }
-  - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiK8yYWLhfZ18WLDvTuBX1YJX1Ll7UMNJqYNES5Eg" }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" }
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
   - do:
@@ -301,7 +301,7 @@ dynamic templates:
                     field: _tsid
 
   - length: { aggregations.filterA.tsids.buckets: 1 }
-  - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiK8yYWLhfZ18WLDvTuBX1YJX1Ll7UMNJqYNES5Eg" }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" }
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
   - do:
@@ -313,14 +313,14 @@ dynamic templates:
             filterA:
               filter:
                 term:
-                  another.dim2: 10
+                  another.dim2: 10.5
               aggs:
                 tsids:
                   terms:
                     field: _tsid
 
   - length: { aggregations.filterA.tsids.buckets: 1 }
-  - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiK8yYWLhfZ18WLDvTuBX1YJX1Ll7UMNJqYNES5Eg" }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" }
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
 ---
@@ -466,13 +466,13 @@ dynamic templates with nesting:
         refresh: true
         body:
           - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
-          - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10" }'
+          - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10.5" }'
           - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
-          - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10" }'
+          - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10.5" }'
           - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
-          - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20" }'
+          - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20.5" }'
           - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
-          - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20" }'
+          - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20.5" }'
 
   - do:
       search:
@@ -498,7 +498,7 @@ dynamic templates with nesting:
                     field: _tsid
 
   - length: { aggregations.filterA.tsids.buckets: 1 }
-  - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK7zJhYuIGKYiosO9O4X2dfFtp-JEbk39FSSMEq_vwX7uw" }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK6pWmkqIGKYiosO9O4X2dfFL8p_4TfsFAUUYYv9EqSmEQ" }
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
   - do:
@@ -517,7 +517,7 @@ dynamic templates with nesting:
                     field: _tsid
 
   - length: { aggregations.filterA.tsids.buckets: 1 }
-  - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK7zJhYuIGKYiosO9O4X2dfFtp-JEbk39FSSMEq_vwX7uw" }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK6pWmkqIGKYiosO9O4X2dfFL8p_4TfsFAUUYYv9EqSmEQ" }
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
   - do:
@@ -536,7 +536,7 @@ dynamic templates with nesting:
                     field: _tsid
 
   - length: { aggregations.filterA.tsids.buckets: 1 }
-  - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK7zJhYuIGKYiosO9O4X2dfFtp-JEbk39FSSMEq_vwX7uw" }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK6pWmkqIGKYiosO9O4X2dfFL8p_4TfsFAUUYYv9EqSmEQ" }
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
   - do:
@@ -548,14 +548,14 @@ dynamic templates with nesting:
             filterA:
               filter:
                 term:
-                  another.dim2: 10
+                  another.dim2: 10.5
               aggs:
                 tsids:
                   terms:
                     field: _tsid
 
   - length: { aggregations.filterA.tsids.buckets: 1 }
-  - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK7zJhYuIGKYiosO9O4X2dfFtp-JEbk39FSSMEq_vwX7uw" }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK6pWmkqIGKYiosO9O4X2dfFL8p_4TfsFAUUYYv9EqSmEQ" }
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
 ---

diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml
@@ -90,11 +90,11 @@ missing routing path field:
 ---
 missing dimension on routing path field:
   - skip:
-      version: " - 8.7.99"
-      reason: error message changed in 8.8.0
+      version: " - 8.13.99"
+      reason: error message changed in 8.14.0
 
   - do:
-      catch: '/All fields that match routing_path must be keywords with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[tag\] was not a dimension./'
+      catch: '/All fields that match routing_path must be configured with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[tag\] was not a dimension./'
       indices.create:
         index: test
         body:

diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml
@@ -121,11 +121,11 @@ top level wildcard dim object:
 ---
 exact match object type:
   - skip:
-      version: " - 8.7.99"
-      reason: routing_path error message updated in 8.8.0
+      version: " - 8.13.99"
+      reason: routing_path error message updated in 8.14.0
 
   - do:
-      catch: '/All fields that match routing_path must be keywords with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[dim\] was \[object\]./'
+      catch: '/All fields that match routing_path must be configured with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[dim\] was \[object\]./'
       indices.create:
         index: tsdb_index
         body:
@@ -154,11 +154,11 @@ exact match object type:
 ---
 non keyword matches routing_path:
   - skip:
-      version: " - 8.7.99"
-      reason: routing_path error message updated in 8.8.0
+      version: " - 8.13.99"
+      reason: routing_path error message updated in 8.14.0
 
   - do:
-      catch: '/All fields that match routing_path must be keywords with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[@timestamp\] was \[date\]./'
+      catch: '/All fields that match routing_path must be configured with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[@timestamp\] was not a dimension./'
       indices.create:
           index: test_index
           body:
@@ -273,7 +273,7 @@ runtime field matching routing path:
         body:
           - '{"index": {}}'
           - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim_kw": "dim", "dim": {"foo": "a"}, "extra_field": 100}'
-  - match: {items.0.index.error.reason: "All fields that match routing_path must be keywords with [time_series_dimension: true] or flattened fields with a list of dimensions in [time_series_dimensions] and without the [script] parameter. [dim.foo] was a runtime [keyword]."}
+  - match: {items.0.index.error.reason: "All fields that match routing_path must be configured with [time_series_dimension: true] or flattened fields with a list of dimensions in [time_series_dimensions] and without the [script] parameter. [dim.foo] was a runtime [keyword]."}
 
 ---
 "dynamic: false matches routing_path":

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java
@@ -208,7 +208,7 @@ protected final LeafFactory leafFactory(SearchExecutionContext context) {
     public void validateMatchedRoutingPath(final String routingPath) {
         throw new IllegalArgumentException(
             "All fields that match routing_path "
-                + "must be keywords with [time_series_dimension: true] "
+                + "must be configured with [time_series_dimension: true] "
                 + "or flattened fields with a list of dimensions in [time_series_dimensions] "
                 + "and without the [script] parameter. ["
                 + name()

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
@@ -125,7 +125,7 @@ public void validate(IndexSettings settings, boolean checkLimits) {
                 // object type is not allowed in the routing paths
                 if (path.equals(objectName)) {
                     throw new IllegalArgumentException(
-                        "All fields that match routing_path must be keywords with [time_series_dimension: true] "
+                        "All fields that match routing_path must be configured with [time_series_dimension: true] "
                             + "or flattened fields with a list of dimensions in [time_series_dimensions] "
                             + "and without the [script] parameter. ["
                             + objectName