TSDB: Add time series information to field caps (#78790)

Exposes information about dimensions and metrics via field caps. This
information will be needed for PromQL support.

Relates to #74660

docs/reference/search/field-caps.asciidoc

@@ -111,6 +111,12 @@ field types are all described as the `keyword` type family.
 `aggregatable`::
   Whether this field can be aggregated on all indices.
 
+`time_series_dimension`::
+  Whether this field is used as a time series dimension.
+
+`time_series_metric`::
+  Contains metric type if this fields is used as a time series metrics, absent if the field is not used as metric.
+
 `indices`::
   The list of indices where this field has the same type family, or null if all indices
   have the same type family for the field.
@@ -123,6 +129,14 @@ field types are all described as the `keyword` type family.
   The list of indices where this field is not aggregatable, or null if all
   indices have the same definition for the field.
 
+`non_dimension_indices`::
+  If this list is present in response then some indices have the field marked as a dimension and other indices, the
+  ones in this list, do not.
+
+`metric_conflicts_indices`::
+  The list of indices where this field is present if these indices don't have the same `time_series_metric` value for
+  this field.
+
 `meta`::
   Merged metadata across all indices as a map of string keys to arrays of values.
   A value length of 1 indicates that all indices had the same value for this key,
@@ -179,7 +193,6 @@ The API returns the following response:
         "metadata_field": false,
         "searchable": true,
         "aggregatable": false
-
       }
     }
   }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/40_time_series.yml

@@ -0,0 +1,231 @@
+---
+setup:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0
+
+  - do:
+      indices.create:
+        index: tsdb_index1
+        body:
+          settings:
+            index:
+              number_of_replicas: 0
+              number_of_shards: 2
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              metricset:
+                type: keyword
+                time_series_dimension: true
+              non_tsdb_field:
+                type: keyword
+              k8s:
+                properties:
+                  pod:
+                    properties:
+                      availability_zone:
+                        type: short
+                        time_series_dimension: true
+                      uid:
+                        type: keyword
+                        time_series_dimension: true
+                      name:
+                        type: keyword
+                      ip:
+                        type: ip
+                        time_series_dimension: true
+                      network:
+                        properties:
+                          tx:
+                            type: long
+                            time_series_metric: counter
+                          rx:
+                            type: integer
+                            time_series_metric: gauge
+                          packets_dropped:
+                            type: long
+                            time_series_metric: gauge
+                          latency:
+                            type: double
+                            time_series_metric: gauge
+
+  - do:
+      indices.create:
+        index: tsdb_index2
+        body:
+          settings:
+            index:
+              number_of_replicas: 0
+              number_of_shards: 2
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              metricset:
+                type: keyword
+              non_tsdb_field:
+                type: keyword
+              k8s:
+                properties:
+                  pod:
+                    properties:
+                      availability_zone:
+                        type: short
+                        time_series_dimension: true
+                      uid:
+                        type: keyword
+                        time_series_dimension: true
+                      name:
+                        type: keyword
+                      ip:
+                        type: ip
+                        time_series_dimension: true
+                      network:
+                        properties:
+                          tx:
+                            type: long
+                            time_series_metric: gauge
+                          rx:
+                            type: integer
+                          packets_dropped:
+                            type: long
+                            time_series_metric: gauge
+                          latency:
+                            type: double
+                            time_series_metric: gauge
+
+---
+"Get simple time series field caps":
+
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0
+
+  - do:
+      field_caps:
+        index: 'tsdb_index1'
+        fields: [ "metricset", "non_tsdb_field", "k8s.pod.*" ]
+
+  - match: {fields.metricset.keyword.searchable: true}
+  - match: {fields.metricset.keyword.aggregatable: true}
+  - match: {fields.metricset.keyword.time_series_dimension: true}
+  - is_false: fields.metricset.keyword.time_series_metric
+  - is_false: fields.metricset.keyword.indices
+  - is_false: fields.metricset.keyword.non_searchable_indices
+  - is_false: fields.metricset.keyword.non_aggregatable_indices
+  - is_false: fields.metricset.keyword.non_dimension_indices
+
+  - match: {fields.non_tsdb_field.keyword.searchable: true}
+  - match: {fields.non_tsdb_field.keyword.aggregatable: true}
+  - is_false: fields.non_tsdb_field.keyword.time_series_dimension
+  - is_false: fields.non_tsdb_field.keyword.time_series_metric
+  - is_false: fields.non_tsdb_field.keyword.indices
+  - is_false: fields.non_tsdb_field.keyword.non_searchable_indices
+  - is_false: fields.non_tsdb_field.keyword.non_aggregatable_indices
+  - is_false: fields.non_tsdb_field.keyword.non_dimension_indices
+
+  - match: {fields.k8s\.pod\.availability_zone.short.time_series_dimension: true}
+  - is_false: fields.k8s\.pod\.availability_zone.short.time_series_metric
+  - is_false: fields.k8s\.pod\.availability_zone.short.non_dimension_indices
+
+  - match: {fields.k8s\.pod\.uid.keyword.time_series_dimension: true}
+  - is_false: fields.k8s\.pod\.uid.keyword.time_series_metric
+  - is_false: fields.k8s\.pod\.uid.keyword.non_dimension_indices
+
+  - is_false: fields.k8s\.pod\.name.keyword.time_series_dimension
+  - is_false: fields.k8s\.pod\.name.keyword.time_series_metric
+  - is_false: fields.k8s\.pod\.name.keyword.non_dimension_indices
+
+  - match: {fields.k8s\.pod\.ip.ip.time_series_dimension: true}
+  - is_false: fields.k8s\.pod\.ip.ip.time_series_metric
+  - is_false: fields.k8s\.pod\.ip.ip.non_dimension_indices
+
+  - is_false: fields.k8s\.pod\.network\.tx.long.time_series_dimension
+  - match: {fields.k8s\.pod\.network\.tx.long.time_series_metric: counter}
+  - is_false: fields.k8s\.pod\.network\.tx.long.non_dimension_indices
+
+  - is_false: fields.k8s\.pod\.network\.rx.integer.time_series_dimension
+  - match: {fields.k8s\.pod\.network\.rx.integer.time_series_metric: gauge}
+  - is_false: fields.k8s\.pod\.network\.rx.integer.non_dimension_indices
+
+  - is_false: fields.k8s\.pod\.network\.packets_dropped.long.time_series_dimension
+  - match: {fields.k8s\.pod\.network\.packets_dropped.long.time_series_metric: gauge}
+  - is_false: fields.k8s\.pod\.network\.packets_dropped.long.non_dimension_indices
+
+  - is_false: fields.k8s\.pod\.network\.latency.double.time_series_dimension
+  - match: {fields.k8s\.pod\.network\.latency.double.time_series_metric: gauge}
+  - is_false: fields.k8s\.pod\.network\.latency.double.non_dimension_indices
+
+---
+"Get time series field caps with conflicts":
+
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0
+
+  - do:
+      field_caps:
+        index: tsdb_index1,tsdb_index2
+        fields: [ "metricset", "non_tsdb_field", "k8s.pod.*" ]
+
+  - match: {fields.metricset.keyword.searchable: true}
+  - match: {fields.metricset.keyword.aggregatable: true}
+  - is_false: fields.metricset.keyword.time_series_dimension
+  - is_false: fields.metricset.keyword.time_series_metric
+  - is_false: fields.metricset.keyword.indices
+  - is_false: fields.metricset.keyword.non_searchable_indices
+  - is_false: fields.metricset.keyword.non_aggregatable_indices
+  - match: {fields.metricset.keyword.non_dimension_indices: ["tsdb_index2"]}
+  - is_false: fields.metricset.keyword.mertric_conflicts_indices
+
+  - match: {fields.non_tsdb_field.keyword.searchable: true}
+  - match: {fields.non_tsdb_field.keyword.aggregatable: true}
+  - is_false: fields.non_tsdb_field.keyword.time_series_dimension
+  - is_false: fields.non_tsdb_field.keyword.time_series_metric
+  - is_false: fields.non_tsdb_field.keyword.indices
+  - is_false: fields.non_tsdb_field.keyword.non_searchable_indices
+  - is_false: fields.non_tsdb_field.keyword.non_aggregatable_indices
+  - is_false: fields.non_tsdb_field.keyword.non_dimension_indices
+  - is_false: fields.non_tsdb_field.keyword.mertric_conflicts_indices
+
+  - match: {fields.k8s\.pod\.availability_zone.short.time_series_dimension: true}
+  - is_false: fields.k8s\.pod\.availability_zone.short.time_series_metric
+  - is_false: fields.k8s\.pod\.availability_zone.short.non_dimension_indices
+  - is_false: fields.k8s\.pod\.availability_zone.short.mertric_conflicts_indices
+
+  - match: {fields.k8s\.pod\.uid.keyword.time_series_dimension: true}
+  - is_false: fields.k8s\.pod\.uid.keyword.time_series_metric
+  - is_false: fields.k8s\.pod\.uid.keyword.non_dimension_indices
+  - is_false: fields.k8s\.pod\.uid.keyword.mertric_conflicts_indices
+
+  - is_false: fields.k8s\.pod\.name.keyword.time_series_dimension
+  - is_false: fields.k8s\.pod\.name.keyword.time_series_metric
+  - is_false: fields.k8s\.pod\.name.keyword.non_dimension_indices
+  - is_false: fields.k8s\.pod\.name.keyword.mertric_conflicts_indices
+
+  - match: {fields.k8s\.pod\.ip.ip.time_series_dimension: true}
+  - is_false: fields.k8s\.pod\.ip.ip.time_series_metric
+  - is_false: fields.k8s\.pod\.ip.ip.non_dimension_indices
+  - is_false: fields.k8s\.pod\.ip.ip.mertric_conflicts_indices
+
+  - is_false: fields.k8s\.pod\.network\.tx.long.time_series_dimension
+  - is_false: fields.k8s\.pod\.network\.tx.long.time_series_metric
+  - is_false: fields.k8s\.pod\.network\.tx.long.non_dimension_indices
+  - match: {fields.k8s\.pod\.network\.tx.long.mertric_conflicts_indices: ["tsdb_index1", "tsdb_index2"]}
+
+  - is_false: fields.k8s\.pod\.network\.rx.integer.time_series_dimension
+  - is_false: fields.k8s\.pod\.network\.rx.integer.time_series_metric
+  - is_false: fields.k8s\.pod\.network\.rx.integer.non_dimension_indices
+  - match: {fields.k8s\.pod\.network\.rx.integer.mertric_conflicts_indices: ["tsdb_index1", "tsdb_index2"]}
+
+  - is_false: fields.k8s\.pod\.network\.packets_dropped.long.time_series_dimension
+  - match: {fields.k8s\.pod\.network\.packets_dropped.long.time_series_metric: gauge}
+  - is_false: fields.k8s\.pod\.network\.packets_dropped.long.non_dimension_indices
+  - is_false: fields.k8s\.pod\.network\.packets_dropped.long.mertric_conflicts_indices
+
+  - is_false: fields.k8s\.pod\.network\.latency.double.time_series_dimension
+  - match: {fields.k8s\.pod\.network\.latency.double.time_series_metric: gauge}
+  - is_false: fields.k8s\.pod\.network\.latency.double.non_dimension_indices
+  - is_false: fields.k8s\.pod\.network\.latency.double.mertric_conflicts_indices

server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java

@@ -15,11 +15,10 @@
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.index.mapper.DocumentParserContext;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
+import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -30,6 +29,8 @@
 import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.transport.RemoteTransportException;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -45,8 +46,10 @@
 
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.array;
 import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
 
 public class FieldCapabilitiesIT extends ESIntegTestCase {
 
@@ -69,6 +72,14 @@ public void setUp() throws Exception {
                         .startObject("playlist")
                             .field("type", "text")
                         .endObject()
+                        .startObject("some_dimension")
+                            .field("type", "keyword")
+                            .field("time_series_dimension", true)
+                        .endObject()
+                        .startObject("some_metric")
+                            .field("type", "long")
+                            .field("time_series_metric", TimeSeriesParams.MetricType.counter)
+                        .endObject()
                         .startObject("secret_soundtrack")
                             .field("type", "alias")
                             .field("path", "playlist")
@@ -98,6 +109,13 @@ public void setUp() throws Exception {
                         .startObject("new_field")
                             .field("type", "long")
                         .endObject()
+                        .startObject("some_dimension")
+                            .field("type", "keyword")
+                        .endObject()
+                        .startObject("some_metric")
+                            .field("type", "long")
+                            .field("time_series_metric", TimeSeriesParams.MetricType.gauge)
+                        .endObject()
                     .endObject()
                 .endObject()
             .endObject();
@@ -285,6 +303,25 @@ public void testWithRunntimeMappings() throws InterruptedException {
         assertTrue(runtimeField.get("keyword").isAggregatable());
     }
 
+    public void testFieldMetricsAndDimensions() {
+        FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index").setFields("some_dimension", "some_metric").get();
+        assertIndices(response, "old_index");
+        assertEquals(2, response.get().size());
+        assertTrue(response.get().containsKey("some_dimension"));
+        assertTrue(response.get().get("some_dimension").get("keyword").isDimension());
+        assertNull(response.get().get("some_dimension").get("keyword").nonDimensionIndices());
+        assertTrue(response.get().containsKey("some_metric"));
+        assertEquals(TimeSeriesParams.MetricType.counter, response.get().get("some_metric").get("long").getMetricType());
+        assertNull(response.get().get("some_metric").get("long").metricConflictsIndices());
+
+        response = client().prepareFieldCaps("old_index", "new_index").setFields("some_dimension", "some_metric").get();
+        assertIndices(response, "old_index", "new_index");
+        assertEquals(2, response.get().size());
+        assertTrue(response.get().containsKey("some_dimension"));
+        assertFalse(response.get().get("some_dimension").get("keyword").isDimension());
+        assertThat(response.get().get("some_dimension").get("keyword").nonDimensionIndices(), array(equalTo("new_index")));
+    }
+
     public void testFailures() throws InterruptedException {
         // in addition to the existing "old_index" and "new_index", create two where the test query throws an error on rewrite
         assertAcked(prepareCreate("index1-error"));