TSDB: Add _tsid field to time_series indices (#80276)

This PR adds support for a field named _tsid that uniquely identifies the time series a document belongs to. When a document is indexed in a time series index (IndexMode.TIME_SERIES), _tsid field is generated from the values of all dimension fields.
elastic · Nov 29, 2021 · c0b4b60 · c0b4b60
1 parent 5cb5d92
commit c0b4b60
Show file tree

Hide file tree

Showing 30 changed files with 1,308 additions and 162 deletions.
diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
@@ -225,7 +225,7 @@ public void testNewReplicas() throws Exception {
     }
 
     public void testSearchTimeSeriesMode() throws Exception {
-        assumeTrue("time series mode introduced in 8.0.0", getOldClusterVersion().onOrAfter(Version.V_8_0_0));
+        assumeTrue("time series index sort by _tsid introduced in 8.1.0", getOldClusterVersion().onOrAfter(Version.V_8_1_0));
         int numDocs;
         if (isRunningAgainstOldCluster()) {
             numDocs = createTimeSeriesModeIndex(1);
@@ -267,7 +267,7 @@ public void testSearchTimeSeriesMode() throws Exception {
     }
 
     public void testNewReplicasTimeSeriesMode() throws Exception {
-        assumeTrue("time series mode introduced in 8.0.0", getOldClusterVersion().onOrAfter(Version.V_8_0_0));
+        assumeTrue("time series index sort by _tsid introduced in 8.1.0", getOldClusterVersion().onOrAfter(Version.V_8_1_0));
         if (isRunningAgainstOldCluster()) {
             createTimeSeriesModeIndex(0);
         } else {

diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java
@@ -258,7 +258,7 @@ private void bulk(String index, String valueSuffix, int count) throws IOExceptio
     }
 
     public void testTsdb() throws IOException {
-        assumeTrue("tsdb added in 8.0.0", UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_0_0));
+        assumeTrue("sort by _tsid added in 8.1.0", UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_1_0));
 
         StringBuilder bulk = new StringBuilder();
         switch (CLUSTER_TYPE) {
@@ -343,20 +343,9 @@ private void assertTsdbAgg(Matcher<?>... expected) throws IOException {
         Request request = new Request("POST", "/tsdb/_search");
         request.addParameter("size", "0");
         XContentBuilder body = JsonXContent.contentBuilder().startObject();
-        // TODO replace tsid runtime field with real tsid
-        body.startObject("runtime_mappings");
-        {
-            body.startObject("tsid");
-            {
-                body.field("type", "keyword");
-                body.field("script", "emit('dim:' + doc['dim'].value)");
-            }
-            body.endObject();
-        }
-        body.endObject();
         body.startObject("aggs").startObject("tsids");
         {
-            body.startObject("terms").field("field", "tsid").endObject();
+            body.startObject("terms").field("field", "_tsid").endObject();
             body.startObject("aggs").startObject("avg");
             {
                 body.startObject("avg").field("field", "value").endObject();
@@ -367,8 +356,7 @@ private void assertTsdbAgg(Matcher<?>... expected) throws IOException {
         request.setJsonEntity(Strings.toString(body.endObject()));
         ListMatcher tsidsExpected = matchesList();
         for (int d = 0; d < expected.length; d++) {
-            // Object key = Map.of("dim", TSDB_DIMS.get(d)); TODO use this once tsid is real
-            Object key = "dim:" + TSDB_DIMS.get(d);
+            Object key = Map.of("dim", TSDB_DIMS.get(d));
             tsidsExpected = tsidsExpected.item(matchesMap().extraOk().entry("key", key).entry("avg", Map.of("value", expected[d])));
         }
         assertMap(

diff --git a/...lRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml b/...lRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml
@@ -178,3 +178,39 @@ can't shadow metrics:
           runtime_mappings:
             deep.deeper.deepest:
               type: keyword
+
+---
+# Test that _tsid field is not added if an index is not a time-series index
+no _tsid in standard indices:
+  - skip:
+      version: " - 8.00.99"
+      reason: _tsid support introduced in 8.1.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              number_of_replicas: 0
+              number_of_shards: 2
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              metricset:
+                type: keyword
+                time_series_dimension: true
+
+  - do:
+      field_caps:
+        index: test
+        fields: [metricset, _tsid]
+
+  - match: {fields.metricset.keyword.searchable:            true}
+  - match: {fields.metricset.keyword.aggregatable:          true}
+  - match: {fields.metricset.keyword.time_series_dimension: true}
+  - is_false: fields.metricset.keyword.indices
+  - is_false: fields.metricset.keyword.non_searchable_indices
+  - is_false: fields.metricset.keyword.non_aggregatable_indices
+  - is_false: fields._tsid  # _tsid metadata field must not exist in non-time-series indices
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml
@@ -186,6 +186,8 @@ set start_time and end_time:
                 end_time: 1632625792000
           mappings:
             properties:
+              "@timestamp":
+                type: date
               metricset:
                 type: keyword
                 time_series_dimension: true

diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml
@@ -204,6 +204,9 @@ runtime field matching routing path:
               properties:
                 "@timestamp":
                   type: date
+                dim_kw:
+                    type: "keyword"
+                    time_series_dimension: true
                 dim:
                   type: object
                   dynamic: runtime
@@ -214,7 +217,7 @@ runtime field matching routing path:
         index: test
         body:
           - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": "a"}}'
+          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim_kw": "dim", "dim": {"foo": "a"}}'
   - match: {items.0.index.error.reason: "All fields that match routing_path must be keywords with [time_series_dimension: true] and without the [script] parameter. [dim.foo] was a runtime [keyword]."}
 
 ---

diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml
@@ -17,8 +17,8 @@ teardown:
 ---
 "Create a snapshot and then restore it":
   - skip:
-      version: " - 7.99.99"
-      reason: introduced in 8.0.0
+      version: " - 8.00.99"
+      reason: _tsid support introduced in 8.1.0
       features: ["allowed_warnings"]
 
   # Create index
@@ -134,10 +134,13 @@ teardown:
       search:
         index: test_index
         body:
+          fields:
+            - field: _tsid
           query:
             query_string:
               query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat'
 
   - match: {hits.total.value: 1}
   - match: {hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507}
-  # TODO assert the _tsid once we generate it
+  - match: {hits.hits.0.fields._tsid: [ { k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod } ] }
+
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml
@@ -90,7 +90,20 @@ query a metric:
 
   - match: {hits.total.value: 1}
 
-# TODO add test showing that quering _tsid fails
+---
+"query tsid fails":
+  - skip:
+      version: " - 8.00.99"
+      reason: _tsid support introduced in 8.1.0
+
+  - do:
+      catch: /\[_tsid\] is not searchable/
+      search:
+        index: test
+        body:
+          query:
+            term:
+              _tsid: wont't work
 
 ---
 fetch a dimension:
@@ -151,7 +164,24 @@ fetch a tag:
   - match: {hits.hits.0.fields.k8s\.pod\.ip: ['10.10.55.2']}
   - is_false: hits.hits.0.fields._tsid   # tsid isn't fetched by default
 
-# TODO add test to fetch the tsid
+---
+"fetch the tsid":
+  - skip:
+      version: " - 8.00.99"
+      reason: _tsid support introduced in 8.1.0
+
+  - do:
+      search:
+        index: test
+        body:
+          fields:
+            - field: _tsid
+          query:
+            query_string:
+              query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat'
+
+  - match: {hits.total.value: 1}
+  - match: {hits.hits.0.fields._tsid: [{k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}]}
 
 ---
 aggregate a dimension:
@@ -229,23 +259,44 @@ aggregate a tag:
   - match: {aggregations.ips.buckets.2.key: 10.10.55.3}
   - match: {aggregations.ips.buckets.2.doc_count: 4}
 
+---
+"aggregate the tsid":
+  - skip:
+      version: " - 8.00.99"
+      reason: _tsid support introduced in 8.1.0
+
+  - do:
+      search:
+        index: test
+        body:
+          size: 0
+          aggs:
+            tsids:
+              terms:
+                field: _tsid
+                order:
+                  _key: asc
 
-# TODO add a test aggregating the _tsid
+  - match: {hits.total.value: 8}
+  - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}}
+  - match: {aggregations.tsids.buckets.0.doc_count: 4}
+  - match: {aggregations.tsids.buckets.1.key: {k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9, metricset: pod}}
+  - match: {aggregations.tsids.buckets.1.doc_count: 4}
 
 ---
 field capabilities:
   - skip:
-      version: " - 7.99.99"
-      reason: introduced in 8.0.0
+      version: " - 8.00.99"
+      reason: _tsid support introduced in 8.1.0
 
   - do:
       field_caps:
         index: test
-        fields: [k8s.pod.uid, k8s.pod.network.rx, k8s.pod.ip, _tsid]
+        fields: [k8s.pod.uid, k8s.pod.network.rx, k8s.pod.ip, metricset, _tsid]
 
-  # TODO assert time_series_metric and time_series_dimension
-  - match: {fields.k8s\.pod\.uid.keyword.searchable:          true}
-  - match: {fields.k8s\.pod\.uid.keyword.aggregatable:        true}
+  - match: {fields.k8s\.pod\.uid.keyword.searchable:            true}
+  - match: {fields.k8s\.pod\.uid.keyword.aggregatable:          true}
+  - match: {fields.k8s\.pod\.uid.keyword.time_series_dimension: true}
   - is_false: fields.k8s\.pod\.uid.keyword.indices
   - is_false: fields.k8s\.pod\.uid.keyword.non_searchable_indices
   - is_false: fields.k8s\.pod\.uid.keyword.non_aggregatable_indices
@@ -259,4 +310,15 @@ field capabilities:
   - is_false: fields.k8s\.pod\.ip.ip.indices
   - is_false: fields.k8s\.pod\.ip.ip.non_searchable_indices
   - is_false: fields.k8s\.pod\.ip.ip.non_aggregatable_indices
-  # TODO assert tsid once we build it:
+  - match: {fields.metricset.keyword.searchable:            true}
+  - match: {fields.metricset.keyword.aggregatable:          true}
+  - match: {fields.metricset.keyword.time_series_dimension: true}
+  - is_false: fields.metricset.keyword.indices
+  - is_false: fields.metricset.keyword.non_searchable_indices
+  - is_false: fields.metricset.keyword.non_aggregatable_indices
+  - match: {fields._tsid._tsid.metadata_field:                true}
+  - match: {fields._tsid._tsid.searchable:                    false}
+  - match: {fields._tsid._tsid.aggregatable:                  true}
+  - is_false: fields._tsid._tsid.indices
+  - is_false: fields._tsid._tsid.non_searchable_indices
+  - is_false: fields._tsid._tsid.non_aggregatable_indices
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml
@@ -57,13 +57,40 @@ setup:
           - '{"index": {}}'
           - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}'
 
-# TODO search on _tsid in an alias
+---
+search an alias:
+  - skip:
+      version: " - 8.00.99"
+      reason: _tsid support introduced in 8.1.0
+
+  - do:
+      indices.put_alias:
+        index: test
+        name:  test_alias
+
+  - do:
+      search:
+        index: test_alias
+        body:
+          size: 0
+          aggs:
+            tsids:
+              terms:
+                field: _tsid
+                order:
+                  _key: asc
+
+  - match: {hits.total.value: 8}
+  - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}}
+  - match: {aggregations.tsids.buckets.0.doc_count: 4}
+  - match: {aggregations.tsids.buckets.1.key: {k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9, metricset: pod}}
+  - match: {aggregations.tsids.buckets.1.doc_count: 4}
 
 ---
 index into alias:
   - skip:
-      version: " - 7.99.99"
-      reason: introduced in 8.0.0
+      version: " - 8.00.99"
+      reason: _tsid support introduced in 8.1.0
 
   - do:
       indices.put_alias:
@@ -85,4 +112,23 @@ index into alias:
           - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "cow", "uid":"1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39", "ip": "10.10.55.4", "network": {"tx": 1434595272, "rx": 530605511}}}}'
   - match: {errors: false}
 
-  # TODO search on tsid once we generate it
+  - do:
+      search:
+        index: test
+        body:
+          size: 0
+          aggs:
+            tsids:
+              terms:
+                field: _tsid
+                order:
+                  _key: asc
+
+  - match: {hits.total.value: 12}
+  - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39, metricset: pod}}
+  - match: {aggregations.tsids.buckets.0.doc_count: 4}
+  - match: {aggregations.tsids.buckets.1.key: {k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}}
+  - match: {aggregations.tsids.buckets.1.doc_count: 4}
+  - match: {aggregations.tsids.buckets.2.key: {k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9, metricset: pod}}
+  - match: {aggregations.tsids.buckets.2.doc_count: 4}
+