From 20dc681f25536747a232354b09db941cf61bd85a Mon Sep 17 00:00:00 2001 From: Dmitry Leontyev Date: Mon, 22 Sep 2025 17:16:41 +0200 Subject: [PATCH 01/16] ES|QL: Make _tsid available in metadata Add _tsid into the list of available attributes in metadata. Closes #133205 --- .../functions/types/count_distinct.md | 4 ++ .../types/count_distinct_over_time.md | 4 ++ .../functions/types/last_over_time.md | 1 + .../definition/functions/count_distinct.json | 66 +++++++++++++++++++ .../functions/count_distinct_over_time.json | 66 +++++++++++++++++++ .../definition/functions/last_over_time.json | 12 ++++ .../index/mapper/TimeSeriesIdFieldMapper.java | 9 +++ .../mapper/TimeSeriesIdFieldMapperTests.java | 19 ++++++ .../core/expression/MetadataAttribute.java | 3 +- .../xpack/esql/core/type/DataType.java | 2 +- .../xpack/esql/EsqlTestUtils.java | 18 ++++- .../main/resources/k8s-timeseries.csv-spec | 32 +++++++++ .../xpack/esql/action/TimeSeriesIT.java | 17 +++++ .../xpack/esql/action/PositionToXContent.java | 13 +++- .../xpack/esql/action/ResponseValueUtils.java | 9 ++- .../function/aggregate/CountDistinct.java | 5 +- .../aggregate/CountDistinctOverTime.java | 2 +- .../function/aggregate/LastOverTime.java | 25 +++---- .../esql/action/EsqlQueryResponseTests.java | 32 +++++++++ .../function/AbstractAggregationTestCase.java | 2 +- .../function/MultiRowTestCaseSupplier.java | 16 +++++ .../aggregate/CountDistinctTests.java | 6 +- .../function/aggregate/LastOverTimeTests.java | 4 +- 23 files changed, 340 insertions(+), 27 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct.md b/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct.md index 8c25c27806c1b..6078aeaa75f47 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct.md @@ -4,6 +4,10 @@ | field | precision | result | | --- | --- | --- | +| _tsid | integer | long | +| _tsid | long | long | +| _tsid | unsigned_long | long | +| _tsid | | long | | boolean | integer | long | | boolean | long | long | | boolean | unsigned_long | long | diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct_over_time.md index 8c25c27806c1b..6078aeaa75f47 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct_over_time.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct_over_time.md @@ -4,6 +4,10 @@ | field | precision | result | | --- | --- | --- | +| _tsid | integer | long | +| _tsid | long | long | +| _tsid | unsigned_long | long | +| _tsid | | long | | boolean | integer | long | | boolean | long | long | | boolean | unsigned_long | long | diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/last_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/types/last_over_time.md index 18df111586e0a..f8a98d36ce91e 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/last_over_time.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/last_over_time.md @@ -4,6 +4,7 @@ | field | result | | --- | --- | +| _tsid | _tsid | | double | double | | integer | integer | | long | long | diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct.json b/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct.json index 894049ac826fb..bb6dd96dd3593 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct.json @@ -4,6 +4,72 @@ "name" : "count_distinct", "description" : "Returns the approximate number of distinct values.", "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "_tsid", + "optional" : false, + "description" : "Column or literal for which to count the number of distinct values." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "field", + "type" : "_tsid", + "optional" : false, + "description" : "Column or literal for which to count the number of distinct values." + }, + { + "name" : "precision", + "type" : "integer", + "optional" : true, + "description" : "Precision threshold. Refer to <>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "field", + "type" : "_tsid", + "optional" : false, + "description" : "Column or literal for which to count the number of distinct values." + }, + { + "name" : "precision", + "type" : "long", + "optional" : true, + "description" : "Precision threshold. Refer to <>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "field", + "type" : "_tsid", + "optional" : false, + "description" : "Column or literal for which to count the number of distinct values." + }, + { + "name" : "precision", + "type" : "unsigned_long", + "optional" : true, + "description" : "Precision threshold. Refer to <>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000." + } + ], + "variadic" : false, + "returnType" : "long" + }, { "params" : [ { diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct_over_time.json b/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct_over_time.json index c67aa8a4a47e3..eef9480e8ed53 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct_over_time.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct_over_time.json @@ -5,6 +5,72 @@ "description" : "The count of distinct values over time for a field.", "note" : "Available with the TS command in snapshot builds", "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "_tsid", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "field", + "type" : "_tsid", + "optional" : false, + "description" : "" + }, + { + "name" : "precision", + "type" : "integer", + "optional" : true, + "description" : "Precision threshold. Refer to <>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "field", + "type" : "_tsid", + "optional" : false, + "description" : "" + }, + { + "name" : "precision", + "type" : "long", + "optional" : true, + "description" : "Precision threshold. Refer to <>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "field", + "type" : "_tsid", + "optional" : false, + "description" : "" + }, + { + "name" : "precision", + "type" : "unsigned_long", + "optional" : true, + "description" : "Precision threshold. Refer to <>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000." + } + ], + "variadic" : false, + "returnType" : "long" + }, { "params" : [ { diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/last_over_time.json b/docs/reference/query-languages/esql/kibana/definition/functions/last_over_time.json index 53a817f0f00a1..809f70a0f613d 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/last_over_time.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/last_over_time.json @@ -5,6 +5,18 @@ "description" : "The latest value of a field, where recency determined by the `@timestamp` field.", "note" : "Available with the TS command in snapshot builds", "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "_tsid", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "_tsid" + }, { "params" : [ { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 0f3776f1db9bb..58f2f4971fac2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -134,6 +134,15 @@ public Query termQuery(Object value, SearchExecutionContext context) { throw new IllegalArgumentException("[" + NAME + "] is not searchable"); } + @Override + public Object valueForDisplay(Object value) { + if (value == null) { + return null; + } + BytesRef binaryValue = (BytesRef) value; + return TimeSeriesIdFieldMapper.encodeTsid(binaryValue); + } + @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { return new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(name()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index 9d56938f185de..da53e25d379ec 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -768,4 +768,23 @@ public void testParseWithDynamicMappingInvalidRoutingHash() { }); assertThat(failure.getMessage(), equalTo("[5:1] failed to parse: Illegal base64 character 20")); } + + public void testValueForDisplay() throws Exception { + DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "long").field("time_series_dimension", true).endObject(); + })); + + ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "value").field("b", 100)); + BytesRef tsidBytes = doc.rootDoc().getBinaryValue("_tsid"); + assertThat(tsidBytes, not(nullValue())); + + TimeSeriesIdFieldMapper.TimeSeriesIdFieldType fieldType = TimeSeriesIdFieldMapper.FIELD_TYPE; + Object displayValue = fieldType.valueForDisplay(tsidBytes); + Object encodedValue = TimeSeriesIdFieldMapper.encodeTsid(tsidBytes); + + assertThat(displayValue, equalTo(encodedValue)); + assertThat(displayValue.getClass(), is(String.class)); + assertThat(fieldType.valueForDisplay(null), nullValue()); + } } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java index 24daf3e66e220..79b2d9958cc09 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java @@ -45,7 +45,8 @@ public class MetadataAttribute extends TypedAttribute { Map.entry(IgnoredFieldMapper.NAME, new MetadataAttributeConfiguration(DataType.KEYWORD, true)), Map.entry(SourceFieldMapper.NAME, new MetadataAttributeConfiguration(DataType.SOURCE, false)), Map.entry(IndexModeFieldMapper.NAME, new MetadataAttributeConfiguration(DataType.KEYWORD, true)), - Map.entry(SCORE, new MetadataAttributeConfiguration(DataType.DOUBLE, false)) + Map.entry(SCORE, new MetadataAttributeConfiguration(DataType.DOUBLE, false)), + Map.entry(TSID_FIELD, new MetadataAttributeConfiguration(DataType.TSID_DATA_TYPE, false)) ); private record MetadataAttributeConfiguration(DataType dataType, boolean searchable) {} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index b3b36ac0fa160..fe825c978cc48 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -390,7 +390,7 @@ public enum DataType { } private static final Collection TYPES = Arrays.stream(values()) - .filter(d -> d != DOC_DATA_TYPE && d != TSID_DATA_TYPE) + .filter(d -> d != DOC_DATA_TYPE) .sorted(Comparator.comparing(DataType::typeName)) .toList(); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index d6cbc95f9ef6d..9ed1de11b8c7c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -42,6 +42,7 @@ import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.h3.H3; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.mapper.RoutingPathFields; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; @@ -887,8 +888,23 @@ public static Literal randomLiteral(DataType type) { throw new UncheckedIOException(e); } } + case TSID_DATA_TYPE -> { + RoutingPathFields routingPathFields = new RoutingPathFields(null); + + int numDimensions = randomIntBetween(1, 4); + for (int i = 0; i < numDimensions; i++) { + String fieldName = "dim" + i; + if (randomBoolean()) { + routingPathFields.addString(fieldName, randomAlphaOfLength(randomIntBetween(3, 10))); + } else { + routingPathFields.addLong(fieldName, randomLongBetween(1, 1000)); + } + } + + yield routingPathFields.buildHash().toBytesRef(); + } case DENSE_VECTOR -> Arrays.asList(randomArray(10, 10, i -> new Float[10], ESTestCase::randomFloat)); - case UNSUPPORTED, OBJECT, DOC_DATA_TYPE, TSID_DATA_TYPE, PARTIAL_AGG -> throw new IllegalArgumentException( + case UNSUPPORTED, OBJECT, DOC_DATA_TYPE, PARTIAL_AGG -> throw new IllegalArgumentException( "can't make random values for [" + type.typeName() + "]" ); }, type); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec index d1f814f482af5..0e620ea28b8fb 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec @@ -518,3 +518,35 @@ true | two | 2024-05-10T00:18:00.000Z false | two | 2024-05-10T00:20:00.000Z false | two | 2024-05-10T00:22:00.000Z ; + +tsidMetadataAttributeCount +required_capability: metrics_command + +TS k8s METADATA _tsid +| STATS cnt = count_distinct(_tsid) +; + +cnt:long +9 +; + +tsidMetadataAttributeAggregation +required_capability: metrics_command + +TS k8s METADATA _tsid +| STATS cnt = COUNT(_tsid) BY cluster, pod +| SORT cluster +; +ignoreOrder:true + +cnt:long | cluster:keyword | pod:keyword +1 | staging | one +1 | staging | two +1 | staging | three +1 | qa | one +1 | qa | two +1 | qa | three +1 | prod | one +1 | prod | two +1 | prod | three +; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java index 63dd77b2a4474..ae567b620ac35 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java @@ -686,4 +686,21 @@ public void testNullMetricsAreSkipped() { assertEquals("Did not filter nulls on counter type", 50, resp.documentsFound()); } } + + public void testTSIDMetadataAttribute() { + List columns = List.of( + new ColumnInfoImpl("_tsid", DataType.TSID_DATA_TYPE, null), + new ColumnInfoImpl("cluster", DataType.KEYWORD, null) + ); + + try (EsqlQueryResponse resp = run(" TS hosts METADATA _tsid | KEEP _tsid, cluster | LIMIT 1")) { + assertThat(resp.columns(), equalTo(columns)); + + List> values = EsqlTestUtils.getValuesList(resp); + assertThat(values, hasSize(1)); + assertThat(values.getFirst().get(0), Matchers.notNullValue()); + assertThat(values.getFirst().get(1), Matchers.notNullValue()); + } + } + } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java index 965c9571f7547..2e8a6d2941b4f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java @@ -18,6 +18,7 @@ import org.elasticsearch.compute.data.FloatBlock; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -199,8 +200,16 @@ protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Pa return builder.value(((FloatBlock) block).getFloat(valueIndex)); } }; - case DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, SHORT, BYTE, OBJECT, FLOAT, HALF_FLOAT, SCALED_FLOAT, - PARTIAL_AGG -> throw new IllegalArgumentException("can't convert values of type [" + columnInfo.type() + "]"); + case TSID_DATA_TYPE -> new PositionToXContent(block) { + @Override + protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Params params, int valueIndex) + throws IOException { + BytesRef bytesRef = ((BytesRefBlock) block).getBytesRef(valueIndex, scratch); + return builder.value(TimeSeriesIdFieldMapper.encodeTsid(bytesRef)); + } + }; + case DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, SHORT, BYTE, OBJECT, FLOAT, HALF_FLOAT, SCALED_FLOAT, PARTIAL_AGG -> + throw new IllegalArgumentException("can't convert values of type [" + columnInfo.type() + "]"); }; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java index 5d0b60c138710..40f33ad62efa5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java @@ -39,6 +39,7 @@ import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.nanoTimeToString; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.spatialToString; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.versionToString; +import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; /** * Collection of static utility methods for helping transform response data between pages and values. @@ -150,9 +151,13 @@ private static Object valueAt(DataType dataType, Block block, int offset, BytesR throw new UncheckedIOException(e); } } + case TSID_DATA_TYPE -> { + BytesRef val = ((BytesRefBlock) block).getBytesRef(offset, scratch); + yield TimeSeriesIdFieldMapper.encodeTsid(val); + } case DENSE_VECTOR -> ((FloatBlock) block).getFloat(offset); - case SHORT, BYTE, FLOAT, HALF_FLOAT, SCALED_FLOAT, OBJECT, DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, NULL, - PARTIAL_AGG -> throw EsqlIllegalArgumentException.illegalDataType(dataType); + case SHORT, BYTE, FLOAT, HALF_FLOAT, SCALED_FLOAT, OBJECT, DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, NULL, PARTIAL_AGG -> + throw EsqlIllegalArgumentException.illegalDataType(dataType); }; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java index 6ecb279abbede..bbba8e13ea907 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java @@ -65,7 +65,8 @@ public class CountDistinct extends AggregateFunction implements OptionalArgument Map.entry(DataType.KEYWORD, CountDistinctBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.IP, CountDistinctBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.VERSION, CountDistinctBytesRefAggregatorFunctionSupplier::new), - Map.entry(DataType.TEXT, CountDistinctBytesRefAggregatorFunctionSupplier::new) + Map.entry(DataType.TEXT, CountDistinctBytesRefAggregatorFunctionSupplier::new), + Map.entry(DataType.TSID_DATA_TYPE, CountDistinctBytesRefAggregatorFunctionSupplier::new) ); private static final int DEFAULT_PRECISION = 3000; @@ -116,7 +117,7 @@ public CountDistinct( Source source, @Param( name = "field", - type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version" }, + type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version", "_tsid" }, description = "Column or literal for which to count the number of distinct values." ) Expression field, @Param( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctOverTime.java index 2e996085e9fc1..1dadde75ac5ba 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctOverTime.java @@ -49,7 +49,7 @@ public CountDistinctOverTime( Source source, @Param( name = "field", - type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version" } + type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version", "_tsid" } ) Expression field, @Param( optional = true, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java index de8d48b2ca9b8..57a1993802882 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.LastBytesRefByTimestampAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.LastDoubleByTimestampAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.LastFloatByTimestampAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.LastIntByTimestampAggregatorFunctionSupplier; @@ -50,13 +51,13 @@ public class LastOverTime extends TimeSeriesAggregateFunction implements Optiona // TODO: support all types @FunctionInfo( type = FunctionType.TIME_SERIES_AGGREGATE, - returnType = { "long", "integer", "double" }, + returnType = { "long", "integer", "double", "_tsid" }, description = "The latest value of a field, where recency determined by the `@timestamp` field.", appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.UNAVAILABLE) }, note = "Available with the [TS](/reference/query-languages/esql/commands/source-commands.md#esql-ts) command in snapshot builds", examples = { @Example(file = "k8s-timeseries", tag = "last_over_time") } ) - public LastOverTime(Source source, @Param(name = "field", type = { "long", "integer", "double" }) Expression field) { + public LastOverTime(Source source, @Param(name = "field", type = { "long", "integer", "double", "_tsid" }) Expression field) { this(source, field, new UnresolvedAttribute(source, "@timestamp")); } @@ -114,16 +115,15 @@ public DataType dataType() { @Override protected TypeResolution resolveType() { - return isType(field(), dt -> dt.isNumeric() && dt != DataType.UNSIGNED_LONG, sourceText(), DEFAULT, "numeric except unsigned_long") - .and( - isType( - timestamp, - dt -> dt == DataType.DATETIME || dt == DataType.DATE_NANOS, - sourceText(), - SECOND, - "date_nanos or datetime" - ) - ); + return isType( + field(), + dt -> (dt.isNumeric() && dt != DataType.UNSIGNED_LONG) || dt == DataType.TSID_DATA_TYPE, + sourceText(), + DEFAULT, + "numeric except unsigned_long or _tsid" + ).and( + isType(timestamp, dt -> dt == DataType.DATETIME || dt == DataType.DATE_NANOS, sourceText(), SECOND, "date_nanos or datetime") + ); } @Override @@ -136,6 +136,7 @@ public AggregatorFunctionSupplier supplier() { case INTEGER -> new LastIntByTimestampAggregatorFunctionSupplier(); case DOUBLE -> new LastDoubleByTimestampAggregatorFunctionSupplier(); case FLOAT -> new LastFloatByTimestampAggregatorFunctionSupplier(); + case TSID_DATA_TYPE -> new LastBytesRefByTimestampAggregatorFunctionSupplier(); default -> throw EsqlIllegalArgumentException.illegalDataType(type); }; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index 5621e10f5f205..2d5be50e10b04 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -44,6 +44,7 @@ import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.h3.H3; import org.elasticsearch.index.mapper.BlockLoader; +import org.elasticsearch.index.mapper.RoutingPathFields; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.test.AbstractChunkedSerializingTestCase; @@ -283,6 +284,24 @@ private Page randomPage(List columns) { } floatBuilder.endPositionEntry(); } + case TSID_DATA_TYPE -> { + RoutingPathFields routingPathFields = new RoutingPathFields(null); + + int numDimensions = randomIntBetween(1, 4); + for (int i = 0; i < numDimensions; i++) { + String fieldName = "dim" + i; + if (randomBoolean()) { + routingPathFields.addString(fieldName, randomAlphaOfLength(randomIntBetween(3, 10))); + } else { + routingPathFields.addLong(fieldName, randomLongBetween(1, 1000)); + } + } + + + new BytesRef(); + + ((BytesRefBlock.Builder) builder).appendBytesRef(routingPathFields.buildHash().toBytesRef()); + } // default -> throw new UnsupportedOperationException("unsupported data type [" + c + "]"); } return builder.build(); @@ -1250,6 +1269,19 @@ static Page valuesToPage(BlockFactory blockFactory, List columns } floatBuilder.endPositionEntry(); } + case TSID_DATA_TYPE -> { + RoutingPathFields routingPathFields = new RoutingPathFields(null); + routingPathFields.addString("dimStr", randomAlphaOfLength(randomIntBetween(3, 10))); + routingPathFields.addLong("dimLong", randomLongBetween(1, 1000)); + + +// BytesRef bytesRef = ((BytesRefBlock) block).getBytesRef(valueIndex, scratch); +// return builder.value(TimeSeriesIdFieldMapper.encodeTsid(bytesRef)); + + + + ((BytesRefBlock.Builder) builder).appendBytesRef(routingPathFields.buildHash().toBytesRef()); + } } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java index 4b799c4172440..5958ded41fd55 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java @@ -554,7 +554,7 @@ protected static String standardAggregatorName(String prefix, DataType type) { case CARTESIAN_SHAPE -> "CartesianShape"; case GEO_POINT -> "GeoPoint"; case GEO_SHAPE -> "GeoShape"; - case KEYWORD, TEXT, VERSION -> "BytesRef"; + case KEYWORD, TEXT, VERSION, TSID_DATA_TYPE -> "BytesRef"; case DOUBLE, COUNTER_DOUBLE -> "Double"; case INTEGER, COUNTER_INTEGER -> "Int"; case IP -> "Ip"; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java index 3827525d74822..03b0ea4fe449f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java @@ -19,6 +19,7 @@ import org.elasticsearch.h3.H3; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.versionfield.Version; @@ -532,6 +533,21 @@ public static List stringCases(int minRows, int maxRows, Data return cases; } + public static List tsidCases(int minRows, int maxRows) { + List cases = new ArrayList<>(); + + addSuppliers( + cases, + minRows, + maxRows, + "_tsid", + DataType.TSID_DATA_TYPE, + () -> EsqlTestUtils.randomLiteral(DataType.TSID_DATA_TYPE).value() + ); + + return cases; + } + private static void addSuppliers( List cases, int minRows, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java index 530591dc40d0d..867516d71cee1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java @@ -57,7 +57,8 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.ipCases(1, 1000), MultiRowTestCaseSupplier.versionCases(1, 1000), MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.KEYWORD), - MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT) + MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT), + MultiRowTestCaseSupplier.tsidCases(1, 1000) ).flatMap(List::stream).forEach(fieldCaseSupplier -> { // With precision for (var precisionCaseSupplier : precisionSuppliers) { @@ -78,7 +79,8 @@ public static Iterable parameters() { DataType.IP, DataType.VERSION, DataType.KEYWORD, - DataType.TEXT + DataType.TEXT, + DataType.TSID_DATA_TYPE )) { var emptyFieldSupplier = new TestCaseSupplier.TypedDataSupplier( "No rows (" + dataType + ")", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java index 0609346111a58..6d04983f38fcc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java @@ -38,8 +38,8 @@ public static Iterable parameters() { var valuesSuppliers = List.of( MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true), MultiRowTestCaseSupplier.intCases(1, 1000, Integer.MIN_VALUE, Integer.MAX_VALUE, true), - MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true) - + MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true), + MultiRowTestCaseSupplier.tsidCases(1, 1000) ); for (List valuesSupplier : valuesSuppliers) { for (TestCaseSupplier.TypedDataSupplier fieldSupplier : valuesSupplier) { From 27b4a4cc98c15472f7de65e90a9f6b7166c155e6 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 22 Sep 2025 15:35:57 +0000 Subject: [PATCH 02/16] [CI] Auto commit changes from spotless --- .../xpack/esql/action/ResponseValueUtils.java | 2 +- .../xpack/esql/action/EsqlQueryResponseTests.java | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java index 40f33ad62efa5..e013836fde15f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java @@ -20,6 +20,7 @@ import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; +import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; @@ -39,7 +40,6 @@ import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.nanoTimeToString; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.spatialToString; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.versionToString; -import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; /** * Collection of static utility methods for helping transform response data between pages and values. diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index 2d5be50e10b04..838f033fffe3b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -297,7 +297,6 @@ private Page randomPage(List columns) { } } - new BytesRef(); ((BytesRefBlock.Builder) builder).appendBytesRef(routingPathFields.buildHash().toBytesRef()); @@ -1274,11 +1273,8 @@ static Page valuesToPage(BlockFactory blockFactory, List columns routingPathFields.addString("dimStr", randomAlphaOfLength(randomIntBetween(3, 10))); routingPathFields.addLong("dimLong", randomLongBetween(1, 1000)); - -// BytesRef bytesRef = ((BytesRefBlock) block).getBytesRef(valueIndex, scratch); -// return builder.value(TimeSeriesIdFieldMapper.encodeTsid(bytesRef)); - - + // BytesRef bytesRef = ((BytesRefBlock) block).getBytesRef(valueIndex, scratch); + // return builder.value(TimeSeriesIdFieldMapper.encodeTsid(bytesRef)); ((BytesRefBlock.Builder) builder).appendBytesRef(routingPathFields.buildHash().toBytesRef()); } From 43610a7f98a0a31124f2bb7fc4b2bb7c5b115488 Mon Sep 17 00:00:00 2001 From: Dmitry Leontyev Date: Mon, 22 Sep 2025 17:50:47 +0200 Subject: [PATCH 03/16] ES|QL: Make _tsid available in metadata Add _tsid into the list of available attributes in metadata. Closes #133205 --- .../org/elasticsearch/test/ESTestCase.java | 17 +++++++++ .../xpack/esql/EsqlTestUtils.java | 16 +-------- .../esql/action/EsqlQueryResponseTests.java | 35 ++----------------- 3 files changed, 20 insertions(+), 48 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 8150456194b76..a29b55d0cefc6 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -124,6 +124,7 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.analysis.TokenizerFactory; +import org.elasticsearch.index.mapper.RoutingPathFields; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.logging.internal.spi.LoggerFactory; @@ -2915,4 +2916,20 @@ public static ProjectState projectStateWithEmptyProject() { public static ProjectMetadata emptyProject() { return ProjectMetadata.builder(randomProjectIdOrDefault()).build(); } + + public static BytesReference randomTsId() { + RoutingPathFields routingPathFields = new RoutingPathFields(null); + + int numDimensions = randomIntBetween(1, 4); + for (int i = 0; i < numDimensions; i++) { + String fieldName = "dim" + i; + if (randomBoolean()) { + routingPathFields.addString(fieldName, randomAlphaOfLength(randomIntBetween(3, 10))); + } else { + routingPathFields.addLong(fieldName, randomLongBetween(1, 1000)); + } + } + + return routingPathFields.buildHash(); + } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index 9ed1de11b8c7c..e1bac0dfbc3b4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -888,21 +888,7 @@ public static Literal randomLiteral(DataType type) { throw new UncheckedIOException(e); } } - case TSID_DATA_TYPE -> { - RoutingPathFields routingPathFields = new RoutingPathFields(null); - - int numDimensions = randomIntBetween(1, 4); - for (int i = 0; i < numDimensions; i++) { - String fieldName = "dim" + i; - if (randomBoolean()) { - routingPathFields.addString(fieldName, randomAlphaOfLength(randomIntBetween(3, 10))); - } else { - routingPathFields.addLong(fieldName, randomLongBetween(1, 1000)); - } - } - - yield routingPathFields.buildHash().toBytesRef(); - } + case TSID_DATA_TYPE -> ESTestCase.randomTsId().toBytesRef(); case DENSE_VECTOR -> Arrays.asList(randomArray(10, 10, i -> new Float[10], ESTestCase::randomFloat)); case UNSUPPORTED, OBJECT, DOC_DATA_TYPE, PARTIAL_AGG -> throw new IllegalArgumentException( "can't make random values for [" + type.typeName() + "]" diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index 2d5be50e10b04..7be8d27fe0c32 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -44,7 +44,6 @@ import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.h3.H3; import org.elasticsearch.index.mapper.BlockLoader; -import org.elasticsearch.index.mapper.RoutingPathFields; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.test.AbstractChunkedSerializingTestCase; @@ -284,24 +283,7 @@ private Page randomPage(List columns) { } floatBuilder.endPositionEntry(); } - case TSID_DATA_TYPE -> { - RoutingPathFields routingPathFields = new RoutingPathFields(null); - - int numDimensions = randomIntBetween(1, 4); - for (int i = 0; i < numDimensions; i++) { - String fieldName = "dim" + i; - if (randomBoolean()) { - routingPathFields.addString(fieldName, randomAlphaOfLength(randomIntBetween(3, 10))); - } else { - routingPathFields.addLong(fieldName, randomLongBetween(1, 1000)); - } - } - - - new BytesRef(); - - ((BytesRefBlock.Builder) builder).appendBytesRef(routingPathFields.buildHash().toBytesRef()); - } + case TSID_DATA_TYPE -> ((BytesRefBlock.Builder) builder).appendBytesRef(ESTestCase.randomTsId().toBytesRef()); // default -> throw new UnsupportedOperationException("unsupported data type [" + c + "]"); } return builder.build(); @@ -1214,7 +1196,7 @@ static Page valuesToPage(BlockFactory blockFactory, List columns case LONG, COUNTER_LONG -> ((LongBlock.Builder) builder).appendLong(((Number) value).longValue()); case INTEGER, COUNTER_INTEGER -> ((IntBlock.Builder) builder).appendInt(((Number) value).intValue()); case DOUBLE, COUNTER_DOUBLE -> ((DoubleBlock.Builder) builder).appendDouble(((Number) value).doubleValue()); - case KEYWORD, TEXT -> ((BytesRefBlock.Builder) builder).appendBytesRef(new BytesRef(value.toString())); + case KEYWORD, TEXT, TSID_DATA_TYPE -> ((BytesRefBlock.Builder) builder).appendBytesRef(new BytesRef(value.toString())); case UNSUPPORTED -> ((BytesRefBlock.Builder) builder).appendNull(); case IP -> ((BytesRefBlock.Builder) builder).appendBytesRef(stringToIP(value.toString())); case DATETIME -> { @@ -1269,19 +1251,6 @@ static Page valuesToPage(BlockFactory blockFactory, List columns } floatBuilder.endPositionEntry(); } - case TSID_DATA_TYPE -> { - RoutingPathFields routingPathFields = new RoutingPathFields(null); - routingPathFields.addString("dimStr", randomAlphaOfLength(randomIntBetween(3, 10))); - routingPathFields.addLong("dimLong", randomLongBetween(1, 1000)); - - -// BytesRef bytesRef = ((BytesRefBlock) block).getBytesRef(valueIndex, scratch); -// return builder.value(TimeSeriesIdFieldMapper.encodeTsid(bytesRef)); - - - - ((BytesRefBlock.Builder) builder).appendBytesRef(routingPathFields.buildHash().toBytesRef()); - } } } } From 8954568a6dc5ffa52eb511aec6186e70b2ad530c Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 22 Sep 2025 15:59:37 +0000 Subject: [PATCH 04/16] [CI] Auto commit changes from spotless --- .../main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index e1bac0dfbc3b4..1d3f19d130ba1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -42,7 +42,6 @@ import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.h3.H3; import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.mapper.RoutingPathFields; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; From f2e7404e83598a40327eb72acf47d5d7fefd11e0 Mon Sep 17 00:00:00 2001 From: Dmitry Leontyev Date: Mon, 22 Sep 2025 18:22:21 +0200 Subject: [PATCH 05/16] ES|QL: Make _tsid available in metadata Add _tsid into the list of available attributes in metadata. Closes #133205 --- .../xpack/esql/action/EsqlQueryResponseTests.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index 7be8d27fe0c32..8d5ec14a27c36 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -192,7 +192,8 @@ private ColumnInfoImpl randomColumnInfo() { || t == DataType.DATE_PERIOD || t == DataType.TIME_DURATION || t == DataType.PARTIAL_AGG - || t == DataType.AGGREGATE_METRIC_DOUBLE, + || t == DataType.AGGREGATE_METRIC_DOUBLE + || t == DataType.TSID_DATA_TYPE, () -> randomFrom(DataType.types()) ).widenSmallNumeric(); return new ColumnInfoImpl(randomAlphaOfLength(10), type.esType(), randomOriginalTypes()); @@ -283,7 +284,6 @@ private Page randomPage(List columns) { } floatBuilder.endPositionEntry(); } - case TSID_DATA_TYPE -> ((BytesRefBlock.Builder) builder).appendBytesRef(ESTestCase.randomTsId().toBytesRef()); // default -> throw new UnsupportedOperationException("unsupported data type [" + c + "]"); } return builder.build(); @@ -1196,7 +1196,7 @@ static Page valuesToPage(BlockFactory blockFactory, List columns case LONG, COUNTER_LONG -> ((LongBlock.Builder) builder).appendLong(((Number) value).longValue()); case INTEGER, COUNTER_INTEGER -> ((IntBlock.Builder) builder).appendInt(((Number) value).intValue()); case DOUBLE, COUNTER_DOUBLE -> ((DoubleBlock.Builder) builder).appendDouble(((Number) value).doubleValue()); - case KEYWORD, TEXT, TSID_DATA_TYPE -> ((BytesRefBlock.Builder) builder).appendBytesRef(new BytesRef(value.toString())); + case KEYWORD, TEXT -> ((BytesRefBlock.Builder) builder).appendBytesRef(new BytesRef(value.toString())); case UNSUPPORTED -> ((BytesRefBlock.Builder) builder).appendNull(); case IP -> ((BytesRefBlock.Builder) builder).appendBytesRef(stringToIP(value.toString())); case DATETIME -> { From 98434a6ac36fbe9aa28b51060cbbc9063262fcb6 Mon Sep 17 00:00:00 2001 From: Dmitry Leontyev Date: Tue, 23 Sep 2025 13:37:23 +0200 Subject: [PATCH 06/16] ES|QL: Make _tsid available in metadata Add an esql capability for the _tsid metadata field. in metadata. Closes #133205 --- .../org/elasticsearch/test/ESTestCase.java | 17 ----------------- .../xpack/esql/EsqlTestUtils.java | 19 ++++++++++++++++++- .../main/resources/k8s-timeseries.csv-spec | 2 ++ .../xpack/esql/action/EsqlCapabilities.java | 5 +++++ 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index a29b55d0cefc6..8150456194b76 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -124,7 +124,6 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.analysis.TokenizerFactory; -import org.elasticsearch.index.mapper.RoutingPathFields; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.logging.internal.spi.LoggerFactory; @@ -2916,20 +2915,4 @@ public static ProjectState projectStateWithEmptyProject() { public static ProjectMetadata emptyProject() { return ProjectMetadata.builder(randomProjectIdOrDefault()).build(); } - - public static BytesReference randomTsId() { - RoutingPathFields routingPathFields = new RoutingPathFields(null); - - int numDimensions = randomIntBetween(1, 4); - for (int i = 0; i < numDimensions; i++) { - String fieldName = "dim" + i; - if (randomBoolean()) { - routingPathFields.addString(fieldName, randomAlphaOfLength(randomIntBetween(3, 10))); - } else { - routingPathFields.addLong(fieldName, randomLongBetween(1, 1000)); - } - } - - return routingPathFields.buildHash(); - } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index 1d3f19d130ba1..e57aa67baeba1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -42,6 +42,7 @@ import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.h3.H3; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.mapper.RoutingPathFields; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; @@ -887,7 +888,7 @@ public static Literal randomLiteral(DataType type) { throw new UncheckedIOException(e); } } - case TSID_DATA_TYPE -> ESTestCase.randomTsId().toBytesRef(); + case TSID_DATA_TYPE -> randomTsId().toBytesRef(); case DENSE_VECTOR -> Arrays.asList(randomArray(10, 10, i -> new Float[10], ESTestCase::randomFloat)); case UNSUPPORTED, OBJECT, DOC_DATA_TYPE, PARTIAL_AGG -> throw new IllegalArgumentException( "can't make random values for [" + type.typeName() + "]" @@ -905,6 +906,22 @@ static Version randomVersion() { }; } + static BytesReference randomTsId() { + RoutingPathFields routingPathFields = new RoutingPathFields(null); + + int numDimensions = randomIntBetween(1, 4); + for (int i = 0; i < numDimensions; i++) { + String fieldName = "dim" + i; + if (randomBoolean()) { + routingPathFields.addString(fieldName, randomAlphaOfLength(randomIntBetween(3, 10))); + } else { + routingPathFields.addLong(fieldName, randomLongBetween(1, 1000)); + } + } + + return routingPathFields.buildHash(); + } + public static WildcardLike wildcardLike(Expression left, String exp) { return new WildcardLike(EMPTY, left, new WildcardPattern(exp), false); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec index 0e620ea28b8fb..3b80fb615684e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec @@ -521,6 +521,7 @@ false | two | 2024-05-10T00:22:00.000Z tsidMetadataAttributeCount required_capability: metrics_command +required_capability: metadata_tsid_field TS k8s METADATA _tsid | STATS cnt = count_distinct(_tsid) @@ -532,6 +533,7 @@ cnt:long tsidMetadataAttributeAggregation required_capability: metrics_command +required_capability: metadata_tsid_field TS k8s METADATA _tsid | STATS cnt = COUNT(_tsid) BY cluster, pod diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 0d7b7eb456cea..54f100bb0ad8d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1533,6 +1533,11 @@ public enum Cap { */ TS_COMMAND_V0(), + /** + * Support for requesting the "_tsid" metadata field. + */ + METADATA_TSID_FIELD, + ; private final boolean enabled; From 09338d78d49c51dfde4140fa653ae9ff65ee0a15 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 23 Sep 2025 11:44:22 +0000 Subject: [PATCH 07/16] [CI] Update transport version definitions --- server/src/main/resources/transport/upper_bounds/9.2.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index bf1a90e5be4e9..6e7d51d3d3020 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -index_request_include_tsid,9167000 +security_stats_endpoint,9168000 From ec17c135166eebebdf065b455c31279ae6c4ddd3 Mon Sep 17 00:00:00 2001 From: Dmitry Leontyev Date: Tue, 23 Sep 2025 15:30:08 +0200 Subject: [PATCH 08/16] ES|QL: Make _tsid available in metadata Fix docs Closes #133205 --- .../esql/_snippets/operators/types/is_not_null.md | 1 + .../kibana/definition/operators/is_not_null.json | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/docs/reference/query-languages/esql/_snippets/operators/types/is_not_null.md b/docs/reference/query-languages/esql/_snippets/operators/types/is_not_null.md index 0a1179d8b1218..83ccfda44807a 100644 --- a/docs/reference/query-languages/esql/_snippets/operators/types/is_not_null.md +++ b/docs/reference/query-languages/esql/_snippets/operators/types/is_not_null.md @@ -4,6 +4,7 @@ | field | result | | --- | --- | +| _tsid | boolean | | boolean | boolean | | cartesian_point | boolean | | cartesian_shape | boolean | diff --git a/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json b/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json index c3cc8fae2a14e..27d56350004cd 100644 --- a/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json +++ b/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json @@ -6,6 +6,18 @@ "description" : "Returns `false` if the value is `NULL`, `true` otherwise.", "note" : "If a field is only in some documents it will be `NULL` in the documents that did not contain it.", "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "_tsid", + "optional" : false, + "description" : "Value to check. It can be a single- or multi-valued column or an expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { From 35da14aa639b710f75e31b06fe5c4becb7a3ac71 Mon Sep 17 00:00:00 2001 From: Dmitry Leontyev Date: Tue, 23 Sep 2025 16:04:47 +0200 Subject: [PATCH 09/16] ES|QL: Make _tsid available in metadata Add EsqlQueryResponseTests round trip test Closes #133205 --- .../xpack/esql/action/EsqlQueryResponseTests.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index 8d5ec14a27c36..77f73f430868e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -58,6 +58,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.type.UnsupportedEsFieldTests; @@ -68,6 +69,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; +import java.util.Base64; import java.util.Iterator; import java.util.List; import java.util.Locale; @@ -284,6 +286,10 @@ private Page randomPage(List columns) { } floatBuilder.endPositionEntry(); } + case TSID_DATA_TYPE -> { + BytesRef tsIdValue = (BytesRef) EsqlTestUtils.randomLiteral(DataType.TSID_DATA_TYPE).value(); + ((BytesRefBlock.Builder) builder).appendBytesRef(tsIdValue); + } // default -> throw new UnsupportedOperationException("unsupported data type [" + c + "]"); } return builder.build(); @@ -1251,6 +1257,11 @@ static Page valuesToPage(BlockFactory blockFactory, List columns } floatBuilder.endPositionEntry(); } + case TSID_DATA_TYPE -> { + // This has been added just to test a round trip. In reality, TSID should never be taken from XContent + byte[] decode = Base64.getUrlDecoder().decode(value.toString()); + ((BytesRefBlock.Builder) builder).appendBytesRef(new BytesRef(decode)); + } } } } From f14c65d1d92b3e4b1a9bd95734ea84709f52ad8e Mon Sep 17 00:00:00 2001 From: Dmitry Leontyev Date: Tue, 23 Sep 2025 16:14:21 +0200 Subject: [PATCH 10/16] Update docs/changelog/135204.yaml --- docs/changelog/135204.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/135204.yaml diff --git a/docs/changelog/135204.yaml b/docs/changelog/135204.yaml new file mode 100644 index 0000000000000..abd90a3d8a892 --- /dev/null +++ b/docs/changelog/135204.yaml @@ -0,0 +1,6 @@ +pr: 135204 +summary: Make `_tsid` available in metadata +area: "ES|QL, TSDB, ES|QL" +type: enhancement +issues: + - 133205 From 80d6197ba4e9b772dc6d7dfe534a02430c609b43 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 23 Sep 2025 14:26:25 +0000 Subject: [PATCH 11/16] [CI] Update transport version definitions --- server/src/main/resources/transport/upper_bounds/9.2.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index 6e7d51d3d3020..b1209b927d8a5 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -security_stats_endpoint,9168000 +inference_api_openai_embeddings_headers,9169000 From a492b47a1674eaf44578fb6de1750aa0a0b73ac2 Mon Sep 17 00:00:00 2001 From: Dmitry Leontyev Date: Tue, 23 Sep 2025 17:37:36 +0200 Subject: [PATCH 12/16] ES|QL: Make _tsid available in metadata Fix tests and docs Closes #133205 --- docs/changelog/135204.yaml | 2 +- .../esql/_snippets/operators/types/is_null.md | 1 + .../esql/kibana/definition/operators/is_null.json | 12 ++++++++++++ .../src/main/resources/k8s-timeseries.csv-spec | 4 ++-- 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/docs/changelog/135204.yaml b/docs/changelog/135204.yaml index abd90a3d8a892..91f9ad4dfa20b 100644 --- a/docs/changelog/135204.yaml +++ b/docs/changelog/135204.yaml @@ -1,6 +1,6 @@ pr: 135204 summary: Make `_tsid` available in metadata -area: "ES|QL, TSDB, ES|QL" +area: ES|QL type: enhancement issues: - 133205 diff --git a/docs/reference/query-languages/esql/_snippets/operators/types/is_null.md b/docs/reference/query-languages/esql/_snippets/operators/types/is_null.md index 0a1179d8b1218..83ccfda44807a 100644 --- a/docs/reference/query-languages/esql/_snippets/operators/types/is_null.md +++ b/docs/reference/query-languages/esql/_snippets/operators/types/is_null.md @@ -4,6 +4,7 @@ | field | result | | --- | --- | +| _tsid | boolean | | boolean | boolean | | cartesian_point | boolean | | cartesian_shape | boolean | diff --git a/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json b/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json index 08c98810ddede..1bbfc5da78575 100644 --- a/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json +++ b/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json @@ -6,6 +6,18 @@ "description" : "Returns `true` if the value is `NULL`, `false` otherwise.", "note" : "If a field is only in some documents it will be `NULL` in the documents that did not contain it.", "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "_tsid", + "optional" : false, + "description" : "Value to check. It can be a single- or multi-valued column or an expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec index 3b80fb615684e..bd629171e7150 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec @@ -520,7 +520,7 @@ false | two | 2024-05-10T00:22:00.000Z ; tsidMetadataAttributeCount -required_capability: metrics_command +required_capability: ts_command_v0 required_capability: metadata_tsid_field TS k8s METADATA _tsid @@ -532,7 +532,7 @@ cnt:long ; tsidMetadataAttributeAggregation -required_capability: metrics_command +required_capability: ts_command_v0 required_capability: metadata_tsid_field TS k8s METADATA _tsid From 3e1f1dd4673b5065697bccc480f6187a97541abd Mon Sep 17 00:00:00 2001 From: Dmitry Leontyev Date: Fri, 26 Sep 2025 09:45:43 +0200 Subject: [PATCH 13/16] ES|QL: Make _tsid available in metadata Add cross-cluster IT. Closes #133205 --- .../esql/action/CrossClusterTimeSeriesIT.java | 168 ++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterTimeSeriesIT.java diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterTimeSeriesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterTimeSeriesIT.java new file mode 100644 index 0000000000000..49d27612d7525 --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterTimeSeriesIT.java @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.action; + +import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.index.mapper.DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; + +// @TestLogging(value = "org.elasticsearch.xpack.esql.session:DEBUG", reason = "to better understand planning") +public class CrossClusterTimeSeriesIT extends AbstractCrossClusterTestCase { + + private static final String INDEX_NAME = "hosts"; + + record Doc(String host, String cluster, long timestamp, int requestCount, double cpu, ByteSizeValue memory) {} + + public void testTsIdMetadataInResponse() { + populateTimeSeriesIndex(LOCAL_CLUSTER, INDEX_NAME); + populateTimeSeriesIndex(REMOTE_CLUSTER_1, INDEX_NAME); + + try (EsqlQueryResponse resp = runQuery("TS hosts, cluster-a:hosts METADATA _tsid", Boolean.TRUE)) { + assertNotNull( + resp.columns().stream().map(ColumnInfoImpl::name).filter(name -> name.equalsIgnoreCase("_tsid")).findFirst().orElse(null) + ); + + assertCCSExecutionInfoDetails(resp.getExecutionInfo(), 2); + } + } + + public void testTsIdMetadataInResponseWithFailure() { + populateTimeSeriesIndex(LOCAL_CLUSTER, INDEX_NAME); + populateTimeSeriesIndex(REMOTE_CLUSTER_1, INDEX_NAME); + + try ( + EsqlQueryResponse resp = runQuery( + "TS hosts, cluster-a:hosts METADATA _tsid | WHERE host IS NOT NULL | STATS cnt = count_distinct(_tsid)", + Boolean.TRUE + ) + ) { + List> values = getValuesList(resp); + assertThat(values, hasSize(1)); + assertNotNull(values.getFirst().getFirst()); + assertCCSExecutionInfoDetails(resp.getExecutionInfo(), 2); + } + } + + private void populateTimeSeriesIndex(String clusterAlias, String indexName) { + int numShards = randomIntBetween(1, 5); + String clusterTag = Strings.isEmpty(clusterAlias) ? "local" : clusterAlias; + Settings settings = Settings.builder() + .put("mode", "time_series") + .putList("routing_path", List.of("host", "cluster")) + .put("index.number_of_shards", numShards) + .build(); + + client(clusterAlias).admin() + .indices() + .prepareCreate(indexName) + .setSettings(settings) + .setMapping( + "@timestamp", + "type=date", + "host", + "type=keyword,time_series_dimension=true", + "cluster", + "type=keyword,time_series_dimension=true", + "cpu", + "type=double,time_series_metric=gauge", + "memory", + "type=long,time_series_metric=gauge", + "request_count", + "type=integer,time_series_metric=counter", + "cluster_tag", + "type=keyword" + ) + .get(); + + final List docs = getRandomDocs(); + + for (Doc doc : docs) { + client().prepareIndex(indexName) + .setSource( + "@timestamp", + doc.timestamp, + "host", + doc.host, + "cluster", + doc.cluster, + "cpu", + doc.cpu, + "memory", + doc.memory.getBytes(), + "request_count", + doc.requestCount, + "cluster_tag", + clusterTag + ) + .get(); + } + client().admin().indices().prepareRefresh(indexName).get(); + } + + private List getRandomDocs() { + final List docs = new ArrayList<>(); + + Map hostToClusters = new HashMap<>(); + for (int i = 0; i < 5; i++) { + hostToClusters.put("p" + i, randomFrom("qa", "prod")); + } + long timestamp = DEFAULT_DATE_TIME_FORMATTER.parseMillis("2024-04-15T00:00:00Z"); + + Map requestCounts = new HashMap<>(); + int numDocs = between(20, 100); + for (int i = 0; i < numDocs; i++) { + List hosts = randomSubsetOf(between(1, hostToClusters.size()), hostToClusters.keySet()); + timestamp += between(1, 10) * 1000L; + for (String host : hosts) { + var requestCount = requestCounts.compute(host, (k, curr) -> { + if (curr == null || randomInt(100) <= 20) { + return randomIntBetween(0, 10); + } else { + return curr + randomIntBetween(1, 10); + } + }); + int cpu = randomIntBetween(0, 100); + ByteSizeValue memory = ByteSizeValue.ofBytes(randomIntBetween(1024, 1024 * 1024)); + docs.add(new Doc(host, hostToClusters.get(host), timestamp, requestCount, cpu, memory)); + } + } + + Randomness.shuffle(docs); + + return docs; + } + + private static void assertCCSExecutionInfoDetails(EsqlExecutionInfo executionInfo, int expectedNumClusters) { + assertNotNull(executionInfo); + assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); + assertTrue(executionInfo.isCrossClusterSearch()); + assertThat(executionInfo.getClusters().size(), equalTo(expectedNumClusters)); + + List clusters = executionInfo.clusterAliases().stream().map(executionInfo::getCluster).toList(); + + for (EsqlExecutionInfo.Cluster cluster : clusters) { + assertThat(cluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(cluster.getSkippedShards(), equalTo(0)); + assertThat(cluster.getFailedShards(), equalTo(0)); + } + } + +} From f289285c0160aba97b5113301ad911aab5c91233 Mon Sep 17 00:00:00 2001 From: Dmitry Leontyev Date: Fri, 26 Sep 2025 10:21:12 +0200 Subject: [PATCH 14/16] ES|QL: Make _tsid available in metadata Add cross-cluster IT. Improve UT. Closes #133205 --- .../functions/types/count_distinct.md | 3 - .../types/count_distinct_over_time.md | 4 -- .../definition/functions/count_distinct.json | 54 --------------- .../functions/count_distinct_over_time.json | 66 ------------------- .../aggregate/CountDistinctOverTime.java | 2 +- .../aggregate/CountDistinctOverTimeTests.java | 3 +- .../aggregate/CountDistinctTests.java | 34 +++++++--- 7 files changed, 29 insertions(+), 137 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct.md b/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct.md index 6078aeaa75f47..0992012e461ed 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct.md @@ -4,9 +4,6 @@ | field | precision | result | | --- | --- | --- | -| _tsid | integer | long | -| _tsid | long | long | -| _tsid | unsigned_long | long | | _tsid | | long | | boolean | integer | long | | boolean | long | long | diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct_over_time.md index 6078aeaa75f47..8c25c27806c1b 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct_over_time.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct_over_time.md @@ -4,10 +4,6 @@ | field | precision | result | | --- | --- | --- | -| _tsid | integer | long | -| _tsid | long | long | -| _tsid | unsigned_long | long | -| _tsid | | long | | boolean | integer | long | | boolean | long | long | | boolean | unsigned_long | long | diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct.json b/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct.json index bb6dd96dd3593..8bb5842fd34b7 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct.json @@ -16,60 +16,6 @@ "variadic" : false, "returnType" : "long" }, - { - "params" : [ - { - "name" : "field", - "type" : "_tsid", - "optional" : false, - "description" : "Column or literal for which to count the number of distinct values." - }, - { - "name" : "precision", - "type" : "integer", - "optional" : true, - "description" : "Precision threshold. Refer to <>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000." - } - ], - "variadic" : false, - "returnType" : "long" - }, - { - "params" : [ - { - "name" : "field", - "type" : "_tsid", - "optional" : false, - "description" : "Column or literal for which to count the number of distinct values." - }, - { - "name" : "precision", - "type" : "long", - "optional" : true, - "description" : "Precision threshold. Refer to <>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000." - } - ], - "variadic" : false, - "returnType" : "long" - }, - { - "params" : [ - { - "name" : "field", - "type" : "_tsid", - "optional" : false, - "description" : "Column or literal for which to count the number of distinct values." - }, - { - "name" : "precision", - "type" : "unsigned_long", - "optional" : true, - "description" : "Precision threshold. Refer to <>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000." - } - ], - "variadic" : false, - "returnType" : "long" - }, { "params" : [ { diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct_over_time.json b/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct_over_time.json index eef9480e8ed53..c67aa8a4a47e3 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct_over_time.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct_over_time.json @@ -5,72 +5,6 @@ "description" : "The count of distinct values over time for a field.", "note" : "Available with the TS command in snapshot builds", "signatures" : [ - { - "params" : [ - { - "name" : "field", - "type" : "_tsid", - "optional" : false, - "description" : "" - } - ], - "variadic" : false, - "returnType" : "long" - }, - { - "params" : [ - { - "name" : "field", - "type" : "_tsid", - "optional" : false, - "description" : "" - }, - { - "name" : "precision", - "type" : "integer", - "optional" : true, - "description" : "Precision threshold. Refer to <>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000." - } - ], - "variadic" : false, - "returnType" : "long" - }, - { - "params" : [ - { - "name" : "field", - "type" : "_tsid", - "optional" : false, - "description" : "" - }, - { - "name" : "precision", - "type" : "long", - "optional" : true, - "description" : "Precision threshold. Refer to <>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000." - } - ], - "variadic" : false, - "returnType" : "long" - }, - { - "params" : [ - { - "name" : "field", - "type" : "_tsid", - "optional" : false, - "description" : "" - }, - { - "name" : "precision", - "type" : "unsigned_long", - "optional" : true, - "description" : "Precision threshold. Refer to <>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000." - } - ], - "variadic" : false, - "returnType" : "long" - }, { "params" : [ { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctOverTime.java index 1dadde75ac5ba..2e996085e9fc1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctOverTime.java @@ -49,7 +49,7 @@ public CountDistinctOverTime( Source source, @Param( name = "field", - type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version", "_tsid" } + type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version" } ) Expression field, @Param( optional = true, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctOverTimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctOverTimeTests.java index 5168f75d85658..e1e523de91c3e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctOverTimeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctOverTimeTests.java @@ -25,7 +25,8 @@ public CountDistinctOverTimeTests(@Name("TestCase") Supplier parameters() { - return CountDistinctTests.parameters(); + List suppliers = CountDistinctTests.suppliers(); + return parameterSuppliersFromTypedData(randomizeBytesRefsOffset(suppliers)); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java index 867516d71cee1..ecc73bcc80e8a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java @@ -39,8 +39,30 @@ public CountDistinctTests(@Name("TestCase") Supplier @ParametersFactory public static Iterable parameters() { - var suppliers = new ArrayList(); + List suppliers = suppliers(); + + // Add some extra cases with TSID + MultiRowTestCaseSupplier.tsidCases(1, 1000).forEach(fieldCaseSupplier -> suppliers.add(makeSupplier(fieldCaseSupplier))); + + suppliers.add( + makeSupplier( + new TestCaseSupplier.TypedDataSupplier( + "No rows (" + DataType.TSID_DATA_TYPE + ")", + List::of, + DataType.TSID_DATA_TYPE, + false, + true, + List.of() + ) + ) + ); + + // "No rows" expects 0 here instead of null + return parameterSuppliersFromTypedData(randomizeBytesRefsOffset(suppliers)); + } + protected static List suppliers() { + var suppliers = new ArrayList(); var precisionSuppliers = Stream.of( TestCaseSupplier.intCases(0, 100_000, true), TestCaseSupplier.longCases(0L, 100_000L, true), @@ -57,8 +79,7 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.ipCases(1, 1000), MultiRowTestCaseSupplier.versionCases(1, 1000), MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.KEYWORD), - MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT), - MultiRowTestCaseSupplier.tsidCases(1, 1000) + MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT) ).flatMap(List::stream).forEach(fieldCaseSupplier -> { // With precision for (var precisionCaseSupplier : precisionSuppliers) { @@ -79,8 +100,7 @@ public static Iterable parameters() { DataType.IP, DataType.VERSION, DataType.KEYWORD, - DataType.TEXT, - DataType.TSID_DATA_TYPE + DataType.TEXT )) { var emptyFieldSupplier = new TestCaseSupplier.TypedDataSupplier( "No rows (" + dataType + ")", @@ -99,9 +119,7 @@ public static Iterable parameters() { // Without precision suppliers.add(makeSupplier(emptyFieldSupplier)); } - - // "No rows" expects 0 here instead of null - return parameterSuppliersFromTypedData(randomizeBytesRefsOffset(suppliers)); + return suppliers; } @Override From 62a63e60e182d31848c028de20905f5ad72e8420 Mon Sep 17 00:00:00 2001 From: Dmitry Leontyev Date: Fri, 26 Sep 2025 15:23:26 +0200 Subject: [PATCH 15/16] ES|QL: Make _tsid available in metadata Remove _tsid from docs. Closes #133205 --- .../functions/types/last_over_time.md | 1 - .../_snippets/operators/types/is_not_null.md | 1 - .../esql/_snippets/operators/types/is_null.md | 1 - .../definition/functions/last_over_time.json | 12 --------- .../definition/operators/is_not_null.json | 12 --------- .../kibana/definition/operators/is_null.json | 12 --------- .../function/aggregate/LastOverTime.java | 25 +++++++++---------- .../function/aggregate/LastOverTimeTests.java | 3 +-- .../function/scalar/nulls/IsNotNullTests.java | 2 +- .../function/scalar/nulls/IsNullTests.java | 6 +++-- 10 files changed, 18 insertions(+), 57 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/last_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/types/last_over_time.md index f8a98d36ce91e..18df111586e0a 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/last_over_time.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/last_over_time.md @@ -4,7 +4,6 @@ | field | result | | --- | --- | -| _tsid | _tsid | | double | double | | integer | integer | | long | long | diff --git a/docs/reference/query-languages/esql/_snippets/operators/types/is_not_null.md b/docs/reference/query-languages/esql/_snippets/operators/types/is_not_null.md index 83ccfda44807a..0a1179d8b1218 100644 --- a/docs/reference/query-languages/esql/_snippets/operators/types/is_not_null.md +++ b/docs/reference/query-languages/esql/_snippets/operators/types/is_not_null.md @@ -4,7 +4,6 @@ | field | result | | --- | --- | -| _tsid | boolean | | boolean | boolean | | cartesian_point | boolean | | cartesian_shape | boolean | diff --git a/docs/reference/query-languages/esql/_snippets/operators/types/is_null.md b/docs/reference/query-languages/esql/_snippets/operators/types/is_null.md index 83ccfda44807a..0a1179d8b1218 100644 --- a/docs/reference/query-languages/esql/_snippets/operators/types/is_null.md +++ b/docs/reference/query-languages/esql/_snippets/operators/types/is_null.md @@ -4,7 +4,6 @@ | field | result | | --- | --- | -| _tsid | boolean | | boolean | boolean | | cartesian_point | boolean | | cartesian_shape | boolean | diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/last_over_time.json b/docs/reference/query-languages/esql/kibana/definition/functions/last_over_time.json index 1daa8eddb41e5..0c732d51ea66b 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/last_over_time.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/last_over_time.json @@ -4,18 +4,6 @@ "name" : "last_over_time", "description" : "Calculates the latest value of a field, where recency determined by the `@timestamp` field.", "signatures" : [ - { - "params" : [ - { - "name" : "field", - "type" : "_tsid", - "optional" : false, - "description" : "" - } - ], - "variadic" : false, - "returnType" : "_tsid" - }, { "params" : [ { diff --git a/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json b/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json index 27d56350004cd..c3cc8fae2a14e 100644 --- a/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json +++ b/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json @@ -6,18 +6,6 @@ "description" : "Returns `false` if the value is `NULL`, `true` otherwise.", "note" : "If a field is only in some documents it will be `NULL` in the documents that did not contain it.", "signatures" : [ - { - "params" : [ - { - "name" : "field", - "type" : "_tsid", - "optional" : false, - "description" : "Value to check. It can be a single- or multi-valued column or an expression." - } - ], - "variadic" : false, - "returnType" : "boolean" - }, { "params" : [ { diff --git a/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json b/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json index 1bbfc5da78575..08c98810ddede 100644 --- a/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json +++ b/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json @@ -6,18 +6,6 @@ "description" : "Returns `true` if the value is `NULL`, `false` otherwise.", "note" : "If a field is only in some documents it will be `NULL` in the documents that did not contain it.", "signatures" : [ - { - "params" : [ - { - "name" : "field", - "type" : "_tsid", - "optional" : false, - "description" : "Value to check. It can be a single- or multi-valued column or an expression." - } - ], - "variadic" : false, - "returnType" : "boolean" - }, { "params" : [ { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java index 835e61a613fc0..cd4fc1f801c08 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java @@ -10,7 +10,6 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; -import org.elasticsearch.compute.aggregation.LastBytesRefByTimestampAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.LastDoubleByTimestampAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.LastFloatByTimestampAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.LastIntByTimestampAggregatorFunctionSupplier; @@ -51,12 +50,12 @@ public class LastOverTime extends TimeSeriesAggregateFunction implements Optiona // TODO: support all types @FunctionInfo( type = FunctionType.TIME_SERIES_AGGREGATE, - returnType = { "long", "integer", "double", "_tsid" }, + returnType = { "long", "integer", "double" }, description = "Calculates the latest value of a field, where recency determined by the `@timestamp` field.", appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0") }, examples = { @Example(file = "k8s-timeseries", tag = "last_over_time") } ) - public LastOverTime(Source source, @Param(name = "field", type = { "long", "integer", "double", "_tsid" }) Expression field) { + public LastOverTime(Source source, @Param(name = "field", type = { "long", "integer", "double" }) Expression field) { this(source, field, new UnresolvedAttribute(source, "@timestamp")); } @@ -114,15 +113,16 @@ public DataType dataType() { @Override protected TypeResolution resolveType() { - return isType( - field(), - dt -> (dt.isNumeric() && dt != DataType.UNSIGNED_LONG) || dt == DataType.TSID_DATA_TYPE, - sourceText(), - DEFAULT, - "date_nanos, datetime or _tsid" - ).and( - isType(timestamp, dt -> dt == DataType.DATETIME || dt == DataType.DATE_NANOS, sourceText(), SECOND, "date_nanos or datetime") - ); + return isType(field(), dt -> dt.isNumeric() && dt != DataType.UNSIGNED_LONG, sourceText(), DEFAULT, "numeric except unsigned_long") + .and( + isType( + timestamp, + dt -> dt == DataType.DATETIME || dt == DataType.DATE_NANOS, + sourceText(), + SECOND, + "date_nanos or datetime" + ) + ); } @Override @@ -135,7 +135,6 @@ public AggregatorFunctionSupplier supplier() { case INTEGER -> new LastIntByTimestampAggregatorFunctionSupplier(); case DOUBLE -> new LastDoubleByTimestampAggregatorFunctionSupplier(); case FLOAT -> new LastFloatByTimestampAggregatorFunctionSupplier(); - case TSID_DATA_TYPE -> new LastBytesRefByTimestampAggregatorFunctionSupplier(); default -> throw EsqlIllegalArgumentException.illegalDataType(type); }; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java index 6d04983f38fcc..2fa0001a510ae 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java @@ -38,8 +38,7 @@ public static Iterable parameters() { var valuesSuppliers = List.of( MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true), MultiRowTestCaseSupplier.intCases(1, 1000, Integer.MIN_VALUE, Integer.MAX_VALUE, true), - MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true), - MultiRowTestCaseSupplier.tsidCases(1, 1000) + MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true) ); for (List valuesSupplier : valuesSuppliers) { for (TestCaseSupplier.TypedDataSupplier fieldSupplier : valuesSupplier) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java index 888eed8e3df09..1c570af525690 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java @@ -34,7 +34,7 @@ public IsNotNullTests(@Name("TestCase") Supplier test public static Iterable parameters() { List suppliers = new ArrayList<>(); for (DataType type : DataType.types()) { - if (false == type.isCounter() && false == DataType.isRepresentable(type)) { + if ((false == type.isCounter() && false == DataType.isRepresentable(type)) || type == DataType.TSID_DATA_TYPE) { continue; } if (type != DataType.NULL) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java index 6ccc5de5f6d7b..59a0404909a31 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java @@ -19,6 +19,7 @@ import org.hamcrest.Matcher; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.function.Supplier; @@ -33,8 +34,9 @@ public IsNullTests(@Name("TestCase") Supplier testCas @ParametersFactory public static Iterable parameters() { List suppliers = new ArrayList<>(); - for (DataType type : DataType.types()) { - if (false == type.isCounter() && false == DataType.isRepresentable(type)) { + Collection types = DataType.types(); + for (DataType type : types) { + if ((false == type.isCounter() && false == DataType.isRepresentable(type)) || type == DataType.TSID_DATA_TYPE) { continue; } if (type != DataType.NULL) { From 5b6cc34116295d2289a5f352b91d2c0347b049ca Mon Sep 17 00:00:00 2001 From: Dmitry Leontyev Date: Fri, 26 Sep 2025 16:54:36 +0200 Subject: [PATCH 16/16] ES|QL: Make _tsid available in metadata Remove _tsid from docs. Closes #133205 --- .../functions/types/count_distinct.md | 1 - .../definition/functions/count_distinct.json | 12 --------- .../main/resources/k8s-timeseries.csv-spec | 2 +- .../function/aggregate/LastOverTime.java | 25 ++++++++++--------- .../function/AbstractFunctionTestCase.java | 5 ++++ .../function/aggregate/LastOverTimeTests.java | 3 ++- 6 files changed, 21 insertions(+), 27 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct.md b/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct.md index 0992012e461ed..8c25c27806c1b 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/count_distinct.md @@ -4,7 +4,6 @@ | field | precision | result | | --- | --- | --- | -| _tsid | | long | | boolean | integer | long | | boolean | long | long | | boolean | unsigned_long | long | diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct.json b/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct.json index 8bb5842fd34b7..894049ac826fb 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/count_distinct.json @@ -4,18 +4,6 @@ "name" : "count_distinct", "description" : "Returns the approximate number of distinct values.", "signatures" : [ - { - "params" : [ - { - "name" : "field", - "type" : "_tsid", - "optional" : false, - "description" : "Column or literal for which to count the number of distinct values." - } - ], - "variadic" : false, - "returnType" : "long" - }, { "params" : [ { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec index 954a2ec381928..66f20720d674d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec @@ -543,7 +543,7 @@ required_capability: ts_command_v0 required_capability: metadata_tsid_field TS k8s METADATA _tsid -| STATS cnt = COUNT(_tsid) BY cluster, pod +| STATS cnt = count_distinct(_tsid) BY cluster, pod | SORT cluster ; ignoreOrder:true diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java index cd4fc1f801c08..07c7dc988dcba 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.LastBytesRefByTimestampAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.LastDoubleByTimestampAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.LastFloatByTimestampAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.LastIntByTimestampAggregatorFunctionSupplier; @@ -50,12 +51,12 @@ public class LastOverTime extends TimeSeriesAggregateFunction implements Optiona // TODO: support all types @FunctionInfo( type = FunctionType.TIME_SERIES_AGGREGATE, - returnType = { "long", "integer", "double" }, + returnType = { "long", "integer", "double", "_tsid" }, description = "Calculates the latest value of a field, where recency determined by the `@timestamp` field.", appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0") }, examples = { @Example(file = "k8s-timeseries", tag = "last_over_time") } ) - public LastOverTime(Source source, @Param(name = "field", type = { "long", "integer", "double" }) Expression field) { + public LastOverTime(Source source, @Param(name = "field", type = { "long", "integer", "double", "_tsid" }) Expression field) { this(source, field, new UnresolvedAttribute(source, "@timestamp")); } @@ -113,16 +114,15 @@ public DataType dataType() { @Override protected TypeResolution resolveType() { - return isType(field(), dt -> dt.isNumeric() && dt != DataType.UNSIGNED_LONG, sourceText(), DEFAULT, "numeric except unsigned_long") - .and( - isType( - timestamp, - dt -> dt == DataType.DATETIME || dt == DataType.DATE_NANOS, - sourceText(), - SECOND, - "date_nanos or datetime" - ) - ); + return isType( + field(), + dt -> (dt.isNumeric() && dt != DataType.UNSIGNED_LONG) || dt == DataType.TSID_DATA_TYPE, + sourceText(), + DEFAULT, + "numeric except unsigned_long" + ).and( + isType(timestamp, dt -> dt == DataType.DATETIME || dt == DataType.DATE_NANOS, sourceText(), SECOND, "date_nanos or datetime") + ); } @Override @@ -135,6 +135,7 @@ public AggregatorFunctionSupplier supplier() { case INTEGER -> new LastIntByTimestampAggregatorFunctionSupplier(); case DOUBLE -> new LastDoubleByTimestampAggregatorFunctionSupplier(); case FLOAT -> new LastFloatByTimestampAggregatorFunctionSupplier(); + case TSID_DATA_TYPE -> new LastBytesRefByTimestampAggregatorFunctionSupplier(); default -> throw EsqlIllegalArgumentException.illegalDataType(type); }; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index d566a90660f15..750c186c3275b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -1055,6 +1055,11 @@ private static boolean isAggregation() { * Should this particular signature be hidden from the docs even though we test it? */ static boolean shouldHideSignature(List argTypes, DataType returnType) { + if (returnType == DataType.TSID_DATA_TYPE || argTypes.stream().anyMatch(p -> p.dataType() == DataType.TSID_DATA_TYPE)) { + // TSID is special (for internal use) and we don't document it + return true; + } + for (DataType dt : DataType.UNDER_CONSTRUCTION.keySet()) { if (returnType == dt || argTypes.stream().anyMatch(p -> p.dataType() == dt)) { return true; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java index 2fa0001a510ae..6d04983f38fcc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java @@ -38,7 +38,8 @@ public static Iterable parameters() { var valuesSuppliers = List.of( MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true), MultiRowTestCaseSupplier.intCases(1, 1000, Integer.MIN_VALUE, Integer.MAX_VALUE, true), - MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true) + MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true), + MultiRowTestCaseSupplier.tsidCases(1, 1000) ); for (List valuesSupplier : valuesSuppliers) { for (TestCaseSupplier.TypedDataSupplier fieldSupplier : valuesSupplier) {