diff --git a/docs/changelog/135204.yaml b/docs/changelog/135204.yaml new file mode 100644 index 0000000000000..91f9ad4dfa20b --- /dev/null +++ b/docs/changelog/135204.yaml @@ -0,0 +1,6 @@ +pr: 135204 +summary: Make `_tsid` available in metadata +area: ES|QL +type: enhancement +issues: + - 133205 diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 288ddc116a08e..828e33b55d720 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -134,6 +134,15 @@ public Query termQuery(Object value, SearchExecutionContext context) { throw new IllegalArgumentException("[" + NAME + "] is not searchable"); } + @Override + public Object valueForDisplay(Object value) { + if (value == null) { + return null; + } + BytesRef binaryValue = (BytesRef) value; + return TimeSeriesIdFieldMapper.encodeTsid(binaryValue); + } + @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { return new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(name()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index 9d56938f185de..da53e25d379ec 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -768,4 +768,23 @@ public void testParseWithDynamicMappingInvalidRoutingHash() { }); assertThat(failure.getMessage(), equalTo("[5:1] failed to parse: Illegal base64 character 20")); } + + public void testValueForDisplay() throws Exception { + DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "long").field("time_series_dimension", true).endObject(); + })); + + ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "value").field("b", 100)); + BytesRef tsidBytes = doc.rootDoc().getBinaryValue("_tsid"); + assertThat(tsidBytes, not(nullValue())); + + TimeSeriesIdFieldMapper.TimeSeriesIdFieldType fieldType = TimeSeriesIdFieldMapper.FIELD_TYPE; + Object displayValue = fieldType.valueForDisplay(tsidBytes); + Object encodedValue = TimeSeriesIdFieldMapper.encodeTsid(tsidBytes); + + assertThat(displayValue, equalTo(encodedValue)); + assertThat(displayValue.getClass(), is(String.class)); + assertThat(fieldType.valueForDisplay(null), nullValue()); + } } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java index 24daf3e66e220..79b2d9958cc09 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java @@ -45,7 +45,8 @@ public class MetadataAttribute extends TypedAttribute { Map.entry(IgnoredFieldMapper.NAME, new MetadataAttributeConfiguration(DataType.KEYWORD, true)), Map.entry(SourceFieldMapper.NAME, new MetadataAttributeConfiguration(DataType.SOURCE, false)), Map.entry(IndexModeFieldMapper.NAME, new MetadataAttributeConfiguration(DataType.KEYWORD, true)), - Map.entry(SCORE, new MetadataAttributeConfiguration(DataType.DOUBLE, false)) + Map.entry(SCORE, new MetadataAttributeConfiguration(DataType.DOUBLE, false)), + Map.entry(TSID_FIELD, new MetadataAttributeConfiguration(DataType.TSID_DATA_TYPE, false)) ); private record MetadataAttributeConfiguration(DataType dataType, boolean searchable) {} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index be96a95d6710d..7a7d5e15e5a9e 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -414,7 +414,7 @@ public enum DataType implements Writeable { } private static final Collection TYPES = Arrays.stream(values()) - .filter(d -> d != DOC_DATA_TYPE && d != TSID_DATA_TYPE) + .filter(d -> d != DOC_DATA_TYPE) .sorted(Comparator.comparing(DataType::typeName)) .toList(); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index 64fe2a1a409e9..badf7b1f1d62f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -45,6 +45,7 @@ import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.h3.H3; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.mapper.RoutingPathFields; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; @@ -901,8 +902,9 @@ public static Literal randomLiteral(DataType type) { throw new UncheckedIOException(e); } } + case TSID_DATA_TYPE -> randomTsId().toBytesRef(); case DENSE_VECTOR -> Arrays.asList(randomArray(10, 10, i -> new Float[10], ESTestCase::randomFloat)); - case UNSUPPORTED, OBJECT, DOC_DATA_TYPE, TSID_DATA_TYPE, PARTIAL_AGG -> throw new IllegalArgumentException( + case UNSUPPORTED, OBJECT, DOC_DATA_TYPE, PARTIAL_AGG -> throw new IllegalArgumentException( "can't make random values for [" + type.typeName() + "]" ); }, type); @@ -918,6 +920,22 @@ static Version randomVersion() { }; } + static BytesReference randomTsId() { + RoutingPathFields routingPathFields = new RoutingPathFields(null); + + int numDimensions = randomIntBetween(1, 4); + for (int i = 0; i < numDimensions; i++) { + String fieldName = "dim" + i; + if (randomBoolean()) { + routingPathFields.addString(fieldName, randomAlphaOfLength(randomIntBetween(3, 10))); + } else { + routingPathFields.addLong(fieldName, randomLongBetween(1, 1000)); + } + } + + return routingPathFields.buildHash(); + } + public static WildcardLike wildcardLike(Expression left, String exp) { return new WildcardLike(EMPTY, left, new WildcardPattern(exp), false); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec index 8e8c8b60f4289..66f20720d674d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec @@ -525,3 +525,37 @@ true | two | 2024-05-10T00:18:00.000Z false | two | 2024-05-10T00:20:00.000Z false | two | 2024-05-10T00:22:00.000Z ; + +tsidMetadataAttributeCount +required_capability: ts_command_v0 +required_capability: metadata_tsid_field + +TS k8s METADATA _tsid +| STATS cnt = count_distinct(_tsid) +; + +cnt:long +9 +; + +tsidMetadataAttributeAggregation +required_capability: ts_command_v0 +required_capability: metadata_tsid_field + +TS k8s METADATA _tsid +| STATS cnt = count_distinct(_tsid) BY cluster, pod +| SORT cluster +; +ignoreOrder:true + +cnt:long | cluster:keyword | pod:keyword +1 | staging | one +1 | staging | two +1 | staging | three +1 | qa | one +1 | qa | two +1 | qa | three +1 | prod | one +1 | prod | two +1 | prod | three +; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterTimeSeriesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterTimeSeriesIT.java new file mode 100644 index 0000000000000..49d27612d7525 --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterTimeSeriesIT.java @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.action; + +import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.index.mapper.DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; + +// @TestLogging(value = "org.elasticsearch.xpack.esql.session:DEBUG", reason = "to better understand planning") +public class CrossClusterTimeSeriesIT extends AbstractCrossClusterTestCase { + + private static final String INDEX_NAME = "hosts"; + + record Doc(String host, String cluster, long timestamp, int requestCount, double cpu, ByteSizeValue memory) {} + + public void testTsIdMetadataInResponse() { + populateTimeSeriesIndex(LOCAL_CLUSTER, INDEX_NAME); + populateTimeSeriesIndex(REMOTE_CLUSTER_1, INDEX_NAME); + + try (EsqlQueryResponse resp = runQuery("TS hosts, cluster-a:hosts METADATA _tsid", Boolean.TRUE)) { + assertNotNull( + resp.columns().stream().map(ColumnInfoImpl::name).filter(name -> name.equalsIgnoreCase("_tsid")).findFirst().orElse(null) + ); + + assertCCSExecutionInfoDetails(resp.getExecutionInfo(), 2); + } + } + + public void testTsIdMetadataInResponseWithFailure() { + populateTimeSeriesIndex(LOCAL_CLUSTER, INDEX_NAME); + populateTimeSeriesIndex(REMOTE_CLUSTER_1, INDEX_NAME); + + try ( + EsqlQueryResponse resp = runQuery( + "TS hosts, cluster-a:hosts METADATA _tsid | WHERE host IS NOT NULL | STATS cnt = count_distinct(_tsid)", + Boolean.TRUE + ) + ) { + List> values = getValuesList(resp); + assertThat(values, hasSize(1)); + assertNotNull(values.getFirst().getFirst()); + assertCCSExecutionInfoDetails(resp.getExecutionInfo(), 2); + } + } + + private void populateTimeSeriesIndex(String clusterAlias, String indexName) { + int numShards = randomIntBetween(1, 5); + String clusterTag = Strings.isEmpty(clusterAlias) ? "local" : clusterAlias; + Settings settings = Settings.builder() + .put("mode", "time_series") + .putList("routing_path", List.of("host", "cluster")) + .put("index.number_of_shards", numShards) + .build(); + + client(clusterAlias).admin() + .indices() + .prepareCreate(indexName) + .setSettings(settings) + .setMapping( + "@timestamp", + "type=date", + "host", + "type=keyword,time_series_dimension=true", + "cluster", + "type=keyword,time_series_dimension=true", + "cpu", + "type=double,time_series_metric=gauge", + "memory", + "type=long,time_series_metric=gauge", + "request_count", + "type=integer,time_series_metric=counter", + "cluster_tag", + "type=keyword" + ) + .get(); + + final List docs = getRandomDocs(); + + for (Doc doc : docs) { + client().prepareIndex(indexName) + .setSource( + "@timestamp", + doc.timestamp, + "host", + doc.host, + "cluster", + doc.cluster, + "cpu", + doc.cpu, + "memory", + doc.memory.getBytes(), + "request_count", + doc.requestCount, + "cluster_tag", + clusterTag + ) + .get(); + } + client().admin().indices().prepareRefresh(indexName).get(); + } + + private List getRandomDocs() { + final List docs = new ArrayList<>(); + + Map hostToClusters = new HashMap<>(); + for (int i = 0; i < 5; i++) { + hostToClusters.put("p" + i, randomFrom("qa", "prod")); + } + long timestamp = DEFAULT_DATE_TIME_FORMATTER.parseMillis("2024-04-15T00:00:00Z"); + + Map requestCounts = new HashMap<>(); + int numDocs = between(20, 100); + for (int i = 0; i < numDocs; i++) { + List hosts = randomSubsetOf(between(1, hostToClusters.size()), hostToClusters.keySet()); + timestamp += between(1, 10) * 1000L; + for (String host : hosts) { + var requestCount = requestCounts.compute(host, (k, curr) -> { + if (curr == null || randomInt(100) <= 20) { + return randomIntBetween(0, 10); + } else { + return curr + randomIntBetween(1, 10); + } + }); + int cpu = randomIntBetween(0, 100); + ByteSizeValue memory = ByteSizeValue.ofBytes(randomIntBetween(1024, 1024 * 1024)); + docs.add(new Doc(host, hostToClusters.get(host), timestamp, requestCount, cpu, memory)); + } + } + + Randomness.shuffle(docs); + + return docs; + } + + private static void assertCCSExecutionInfoDetails(EsqlExecutionInfo executionInfo, int expectedNumClusters) { + assertNotNull(executionInfo); + assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); + assertTrue(executionInfo.isCrossClusterSearch()); + assertThat(executionInfo.getClusters().size(), equalTo(expectedNumClusters)); + + List clusters = executionInfo.clusterAliases().stream().map(executionInfo::getCluster).toList(); + + for (EsqlExecutionInfo.Cluster cluster : clusters) { + assertThat(cluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(cluster.getSkippedShards(), equalTo(0)); + assertThat(cluster.getFailedShards(), equalTo(0)); + } + } + +} diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java index 63dd77b2a4474..ae567b620ac35 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java @@ -686,4 +686,21 @@ public void testNullMetricsAreSkipped() { assertEquals("Did not filter nulls on counter type", 50, resp.documentsFound()); } } + + public void testTSIDMetadataAttribute() { + List columns = List.of( + new ColumnInfoImpl("_tsid", DataType.TSID_DATA_TYPE, null), + new ColumnInfoImpl("cluster", DataType.KEYWORD, null) + ); + + try (EsqlQueryResponse resp = run(" TS hosts METADATA _tsid | KEEP _tsid, cluster | LIMIT 1")) { + assertThat(resp.columns(), equalTo(columns)); + + List> values = EsqlTestUtils.getValuesList(resp); + assertThat(values, hasSize(1)); + assertThat(values.getFirst().get(0), Matchers.notNullValue()); + assertThat(values.getFirst().get(1), Matchers.notNullValue()); + } + } + } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 3628eceb4593c..6d6ba59abac19 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1558,9 +1558,12 @@ public enum Cap { INLINE_STATS_FIX_OPTIMIZED_AS_LOCAL_RELATION(INLINESTATS_V11.enabled), - DENSE_VECTOR_AGG_METRIC_DOUBLE_IF_FNS + DENSE_VECTOR_AGG_METRIC_DOUBLE_IF_FNS, - ; + /** + * Support for requesting the "_tsid" metadata field. + */ + METADATA_TSID_FIELD; private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java index 965c9571f7547..2e8a6d2941b4f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java @@ -18,6 +18,7 @@ import org.elasticsearch.compute.data.FloatBlock; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -199,8 +200,16 @@ protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Pa return builder.value(((FloatBlock) block).getFloat(valueIndex)); } }; - case DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, SHORT, BYTE, OBJECT, FLOAT, HALF_FLOAT, SCALED_FLOAT, - PARTIAL_AGG -> throw new IllegalArgumentException("can't convert values of type [" + columnInfo.type() + "]"); + case TSID_DATA_TYPE -> new PositionToXContent(block) { + @Override + protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Params params, int valueIndex) + throws IOException { + BytesRef bytesRef = ((BytesRefBlock) block).getBytesRef(valueIndex, scratch); + return builder.value(TimeSeriesIdFieldMapper.encodeTsid(bytesRef)); + } + }; + case DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, SHORT, BYTE, OBJECT, FLOAT, HALF_FLOAT, SCALED_FLOAT, PARTIAL_AGG -> + throw new IllegalArgumentException("can't convert values of type [" + columnInfo.type() + "]"); }; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java index 5d0b60c138710..e013836fde15f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java @@ -20,6 +20,7 @@ import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; +import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; @@ -150,9 +151,13 @@ private static Object valueAt(DataType dataType, Block block, int offset, BytesR throw new UncheckedIOException(e); } } + case TSID_DATA_TYPE -> { + BytesRef val = ((BytesRefBlock) block).getBytesRef(offset, scratch); + yield TimeSeriesIdFieldMapper.encodeTsid(val); + } case DENSE_VECTOR -> ((FloatBlock) block).getFloat(offset); - case SHORT, BYTE, FLOAT, HALF_FLOAT, SCALED_FLOAT, OBJECT, DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, NULL, - PARTIAL_AGG -> throw EsqlIllegalArgumentException.illegalDataType(dataType); + case SHORT, BYTE, FLOAT, HALF_FLOAT, SCALED_FLOAT, OBJECT, DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, NULL, PARTIAL_AGG -> + throw EsqlIllegalArgumentException.illegalDataType(dataType); }; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java index 6ecb279abbede..bbba8e13ea907 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java @@ -65,7 +65,8 @@ public class CountDistinct extends AggregateFunction implements OptionalArgument Map.entry(DataType.KEYWORD, CountDistinctBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.IP, CountDistinctBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.VERSION, CountDistinctBytesRefAggregatorFunctionSupplier::new), - Map.entry(DataType.TEXT, CountDistinctBytesRefAggregatorFunctionSupplier::new) + Map.entry(DataType.TEXT, CountDistinctBytesRefAggregatorFunctionSupplier::new), + Map.entry(DataType.TSID_DATA_TYPE, CountDistinctBytesRefAggregatorFunctionSupplier::new) ); private static final int DEFAULT_PRECISION = 3000; @@ -116,7 +117,7 @@ public CountDistinct( Source source, @Param( name = "field", - type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version" }, + type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version", "_tsid" }, description = "Column or literal for which to count the number of distinct values." ) Expression field, @Param( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java index cd4fc1f801c08..07c7dc988dcba 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.LastBytesRefByTimestampAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.LastDoubleByTimestampAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.LastFloatByTimestampAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.LastIntByTimestampAggregatorFunctionSupplier; @@ -50,12 +51,12 @@ public class LastOverTime extends TimeSeriesAggregateFunction implements Optiona // TODO: support all types @FunctionInfo( type = FunctionType.TIME_SERIES_AGGREGATE, - returnType = { "long", "integer", "double" }, + returnType = { "long", "integer", "double", "_tsid" }, description = "Calculates the latest value of a field, where recency determined by the `@timestamp` field.", appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0") }, examples = { @Example(file = "k8s-timeseries", tag = "last_over_time") } ) - public LastOverTime(Source source, @Param(name = "field", type = { "long", "integer", "double" }) Expression field) { + public LastOverTime(Source source, @Param(name = "field", type = { "long", "integer", "double", "_tsid" }) Expression field) { this(source, field, new UnresolvedAttribute(source, "@timestamp")); } @@ -113,16 +114,15 @@ public DataType dataType() { @Override protected TypeResolution resolveType() { - return isType(field(), dt -> dt.isNumeric() && dt != DataType.UNSIGNED_LONG, sourceText(), DEFAULT, "numeric except unsigned_long") - .and( - isType( - timestamp, - dt -> dt == DataType.DATETIME || dt == DataType.DATE_NANOS, - sourceText(), - SECOND, - "date_nanos or datetime" - ) - ); + return isType( + field(), + dt -> (dt.isNumeric() && dt != DataType.UNSIGNED_LONG) || dt == DataType.TSID_DATA_TYPE, + sourceText(), + DEFAULT, + "numeric except unsigned_long" + ).and( + isType(timestamp, dt -> dt == DataType.DATETIME || dt == DataType.DATE_NANOS, sourceText(), SECOND, "date_nanos or datetime") + ); } @Override @@ -135,6 +135,7 @@ public AggregatorFunctionSupplier supplier() { case INTEGER -> new LastIntByTimestampAggregatorFunctionSupplier(); case DOUBLE -> new LastDoubleByTimestampAggregatorFunctionSupplier(); case FLOAT -> new LastFloatByTimestampAggregatorFunctionSupplier(); + case TSID_DATA_TYPE -> new LastBytesRefByTimestampAggregatorFunctionSupplier(); default -> throw EsqlIllegalArgumentException.illegalDataType(type); }; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index 5621e10f5f205..77f73f430868e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -58,6 +58,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.type.UnsupportedEsFieldTests; @@ -68,6 +69,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; +import java.util.Base64; import java.util.Iterator; import java.util.List; import java.util.Locale; @@ -192,7 +194,8 @@ private ColumnInfoImpl randomColumnInfo() { || t == DataType.DATE_PERIOD || t == DataType.TIME_DURATION || t == DataType.PARTIAL_AGG - || t == DataType.AGGREGATE_METRIC_DOUBLE, + || t == DataType.AGGREGATE_METRIC_DOUBLE + || t == DataType.TSID_DATA_TYPE, () -> randomFrom(DataType.types()) ).widenSmallNumeric(); return new ColumnInfoImpl(randomAlphaOfLength(10), type.esType(), randomOriginalTypes()); @@ -283,6 +286,10 @@ private Page randomPage(List columns) { } floatBuilder.endPositionEntry(); } + case TSID_DATA_TYPE -> { + BytesRef tsIdValue = (BytesRef) EsqlTestUtils.randomLiteral(DataType.TSID_DATA_TYPE).value(); + ((BytesRefBlock.Builder) builder).appendBytesRef(tsIdValue); + } // default -> throw new UnsupportedOperationException("unsupported data type [" + c + "]"); } return builder.build(); @@ -1250,6 +1257,11 @@ static Page valuesToPage(BlockFactory blockFactory, List columns } floatBuilder.endPositionEntry(); } + case TSID_DATA_TYPE -> { + // This has been added just to test a round trip. In reality, TSID should never be taken from XContent + byte[] decode = Base64.getUrlDecoder().decode(value.toString()); + ((BytesRefBlock.Builder) builder).appendBytesRef(new BytesRef(decode)); + } } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java index 4b799c4172440..5958ded41fd55 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java @@ -554,7 +554,7 @@ protected static String standardAggregatorName(String prefix, DataType type) { case CARTESIAN_SHAPE -> "CartesianShape"; case GEO_POINT -> "GeoPoint"; case GEO_SHAPE -> "GeoShape"; - case KEYWORD, TEXT, VERSION -> "BytesRef"; + case KEYWORD, TEXT, VERSION, TSID_DATA_TYPE -> "BytesRef"; case DOUBLE, COUNTER_DOUBLE -> "Double"; case INTEGER, COUNTER_INTEGER -> "Int"; case IP -> "Ip"; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index d566a90660f15..750c186c3275b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -1055,6 +1055,11 @@ private static boolean isAggregation() { * Should this particular signature be hidden from the docs even though we test it? */ static boolean shouldHideSignature(List argTypes, DataType returnType) { + if (returnType == DataType.TSID_DATA_TYPE || argTypes.stream().anyMatch(p -> p.dataType() == DataType.TSID_DATA_TYPE)) { + // TSID is special (for internal use) and we don't document it + return true; + } + for (DataType dt : DataType.UNDER_CONSTRUCTION.keySet()) { if (returnType == dt || argTypes.stream().anyMatch(p -> p.dataType() == dt)) { return true; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java index 3827525d74822..03b0ea4fe449f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java @@ -19,6 +19,7 @@ import org.elasticsearch.h3.H3; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.versionfield.Version; @@ -532,6 +533,21 @@ public static List stringCases(int minRows, int maxRows, Data return cases; } + public static List tsidCases(int minRows, int maxRows) { + List cases = new ArrayList<>(); + + addSuppliers( + cases, + minRows, + maxRows, + "_tsid", + DataType.TSID_DATA_TYPE, + () -> EsqlTestUtils.randomLiteral(DataType.TSID_DATA_TYPE).value() + ); + + return cases; + } + private static void addSuppliers( List cases, int minRows, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctOverTimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctOverTimeTests.java index 5168f75d85658..e1e523de91c3e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctOverTimeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctOverTimeTests.java @@ -25,7 +25,8 @@ public CountDistinctOverTimeTests(@Name("TestCase") Supplier parameters() { - return CountDistinctTests.parameters(); + List suppliers = CountDistinctTests.suppliers(); + return parameterSuppliersFromTypedData(randomizeBytesRefsOffset(suppliers)); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java index 530591dc40d0d..ecc73bcc80e8a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java @@ -39,8 +39,30 @@ public CountDistinctTests(@Name("TestCase") Supplier @ParametersFactory public static Iterable parameters() { - var suppliers = new ArrayList(); + List suppliers = suppliers(); + + // Add some extra cases with TSID + MultiRowTestCaseSupplier.tsidCases(1, 1000).forEach(fieldCaseSupplier -> suppliers.add(makeSupplier(fieldCaseSupplier))); + + suppliers.add( + makeSupplier( + new TestCaseSupplier.TypedDataSupplier( + "No rows (" + DataType.TSID_DATA_TYPE + ")", + List::of, + DataType.TSID_DATA_TYPE, + false, + true, + List.of() + ) + ) + ); + + // "No rows" expects 0 here instead of null + return parameterSuppliersFromTypedData(randomizeBytesRefsOffset(suppliers)); + } + protected static List suppliers() { + var suppliers = new ArrayList(); var precisionSuppliers = Stream.of( TestCaseSupplier.intCases(0, 100_000, true), TestCaseSupplier.longCases(0L, 100_000L, true), @@ -97,9 +119,7 @@ public static Iterable parameters() { // Without precision suppliers.add(makeSupplier(emptyFieldSupplier)); } - - // "No rows" expects 0 here instead of null - return parameterSuppliersFromTypedData(randomizeBytesRefsOffset(suppliers)); + return suppliers; } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java index 0609346111a58..6d04983f38fcc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java @@ -38,8 +38,8 @@ public static Iterable parameters() { var valuesSuppliers = List.of( MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true), MultiRowTestCaseSupplier.intCases(1, 1000, Integer.MIN_VALUE, Integer.MAX_VALUE, true), - MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true) - + MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true), + MultiRowTestCaseSupplier.tsidCases(1, 1000) ); for (List valuesSupplier : valuesSuppliers) { for (TestCaseSupplier.TypedDataSupplier fieldSupplier : valuesSupplier) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java index 888eed8e3df09..1c570af525690 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java @@ -34,7 +34,7 @@ public IsNotNullTests(@Name("TestCase") Supplier test public static Iterable parameters() { List suppliers = new ArrayList<>(); for (DataType type : DataType.types()) { - if (false == type.isCounter() && false == DataType.isRepresentable(type)) { + if ((false == type.isCounter() && false == DataType.isRepresentable(type)) || type == DataType.TSID_DATA_TYPE) { continue; } if (type != DataType.NULL) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java index 6ccc5de5f6d7b..59a0404909a31 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java @@ -19,6 +19,7 @@ import org.hamcrest.Matcher; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.function.Supplier; @@ -33,8 +34,9 @@ public IsNullTests(@Name("TestCase") Supplier testCas @ParametersFactory public static Iterable parameters() { List suppliers = new ArrayList<>(); - for (DataType type : DataType.types()) { - if (false == type.isCounter() && false == DataType.isRepresentable(type)) { + Collection types = DataType.types(); + for (DataType type : types) { + if ((false == type.isCounter() && false == DataType.isRepresentable(type)) || type == DataType.TSID_DATA_TYPE) { continue; } if (type != DataType.NULL) {