From 31813c20961c657182441e3b03304601af4234bc Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 26 Sep 2025 12:05:34 +0200 Subject: [PATCH 1/3] Add time range bucketing attribute to APM shard search latency metrics We recently added relevant attributes to the existing shard search latency metrics (#134798). This commit introduces an additional attribute that analyzes the parsed time range filter against the @timestamp field and reports back whether it is within the last 15 minutes, last hour, last 12 hours, last day, last three days, last seven days, or last 14 days. --- .../SearchRequestAttributesExtractor.java | 51 +++++++++++-- .../index/mapper/DateFieldMapper.java | 76 +++++++++++-------- .../index/mapper/DateScriptFieldType.java | 1 + .../index/query/SearchExecutionContext.java | 24 ++++++ .../stats/ShardSearchPhaseAPMMetrics.java | 22 +++++- ...SearchRequestAttributesExtractorTests.java | 52 +++++++++++++ .../ShardSearchPhaseAPMMetricsTests.java | 3 +- 7 files changed, 186 insertions(+), 43 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java index fcc52e8892c18..1bf6498ed963f 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java @@ -47,16 +47,18 @@ private SearchRequestAttributesExtractor() {} * Introspects the provided search request and extracts metadata from it about some of its characteristics. */ public static Map extractAttributes(SearchRequest searchRequest, String[] localIndices) { - return extractAttributes(searchRequest.source(), searchRequest.scroll(), localIndices); + return extractAttributes(searchRequest.source(), searchRequest.scroll(), null, -1, localIndices); } /** * Introspects the provided shard search request and extracts metadata from it about some of its characteristics. */ - public static Map extractAttributes(ShardSearchRequest shardSearchRequest) { + public static Map extractAttributes(ShardSearchRequest shardSearchRequest, Long rangeTimestampFrom, long nowInMillis) { Map attributes = extractAttributes( shardSearchRequest.source(), shardSearchRequest.scroll(), + rangeTimestampFrom, + nowInMillis, shardSearchRequest.shardId().getIndexName() ); boolean isSystem = ((EsExecutors.EsThread) Thread.currentThread()).isSystem(); @@ -67,6 +69,8 @@ public static Map extractAttributes(ShardSearchRequest shardSear private static Map extractAttributes( SearchSourceBuilder searchSourceBuilder, TimeValue scroll, + Long rangeTimestampFrom, + long nowInMillis, String... localIndices ) { String target = extractIndices(localIndices); @@ -77,7 +81,7 @@ private static Map extractAttributes( } if (searchSourceBuilder == null) { - return buildAttributesMap(target, ScoreSortBuilder.NAME, HITS_ONLY, false, false, false, pitOrScroll); + return buildAttributesMap(target, ScoreSortBuilder.NAME, HITS_ONLY, false, false, false, pitOrScroll, null); } if (searchSourceBuilder.pointInTimeBuilder() != null) { @@ -103,6 +107,10 @@ private static Map extractAttributes( } final boolean hasKnn = searchSourceBuilder.knnSearch().isEmpty() == false || queryMetadataBuilder.knnQuery; + String timestampRangeFilter = null; + if (rangeTimestampFrom != null) { + timestampRangeFilter = introspectTimeRange(rangeTimestampFrom, nowInMillis); + } return buildAttributesMap( target, primarySort, @@ -110,7 +118,8 @@ private static Map extractAttributes( hasKnn, queryMetadataBuilder.rangeOnTimestamp, queryMetadataBuilder.rangeOnEventIngested, - pitOrScroll + pitOrScroll, + timestampRangeFilter ); } @@ -121,7 +130,8 @@ private static Map buildAttributesMap( boolean knn, boolean rangeOnTimestamp, boolean rangeOnEventIngested, - String pitOrScroll + String pitOrScroll, + String timestampRangeFilter ) { Map attributes = new HashMap<>(5, 1.0f); attributes.put(TARGET_ATTRIBUTE, target); @@ -139,6 +149,9 @@ private static Map buildAttributesMap( if (rangeOnEventIngested) { attributes.put(RANGE_EVENT_INGESTED_ATTRIBUTE, rangeOnEventIngested); } + if (timestampRangeFilter != null) { + attributes.put(TIMESTAMP_RANGE_FILTER_ATTRIBUTE, timestampRangeFilter); + } return attributes; } @@ -155,6 +168,7 @@ private static final class QueryMetadataBuilder { static final String KNN_ATTRIBUTE = "knn"; static final String RANGE_TIMESTAMP_ATTRIBUTE = "range_timestamp"; static final String RANGE_EVENT_INGESTED_ATTRIBUTE = "range_event_ingested"; + static final String TIMESTAMP_RANGE_FILTER_ATTRIBUTE = "timestamp_range_filter"; private static final String TARGET_KIBANA = ".kibana"; private static final String TARGET_ML = ".ml"; @@ -310,4 +324,31 @@ private static void introspectQueryBuilder(QueryBuilder queryBuilder, QueryMetad default: } } + + private enum TimeRangeBucket { + FifteenMinutes(TimeValue.timeValueMinutes(15).getMillis(), "15_minutes"), + OneHour(TimeValue.timeValueHours(1).getMillis(), "1_hour"), + TwelveHours(TimeValue.timeValueHours(12).getMillis(), "12_hours"), + OneDay(TimeValue.timeValueDays(1).getMillis(), "1_day"), + ThreeDays(TimeValue.timeValueDays(3).getMillis(), "3_days"), + SevenDays(TimeValue.timeValueDays(7).getMillis(), "7_days"), + FourteenDays(TimeValue.timeValueDays(14).getMillis(), "14_days"); + + private final long millis; + private final String bucketName; + + TimeRangeBucket(long millis, String bucketName) { + this.millis = millis; + this.bucketName = bucketName; + } + } + + static String introspectTimeRange(long timeRangeFrom, long nowInMillis) { + for (TimeRangeBucket value : TimeRangeBucket.values()) { + if (timeRangeFrom >= nowInMillis - value.millis) { + return value.bucketName; + } + } + return "older_than_14_days"; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index fdb8bad484d74..0e34261baa996 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -25,6 +25,7 @@ import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.Query; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; @@ -747,33 +748,34 @@ public Query rangeQuery( if (relation == ShapeRelation.DISJOINT) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support DISJOINT ranges"); } - DateMathParser parser; - if (forcedDateParser == null) { - if (lowerTerm instanceof Number || upperTerm instanceof Number) { - // force epoch_millis - parser = EPOCH_MILLIS_PARSER; - } else { - parser = dateMathParser; - } - } else { - parser = forcedDateParser; - } - return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> { - Query query; - if (isIndexed()) { - query = LongPoint.newRangeQuery(name(), l, u); - if (hasDocValues()) { - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); - query = new IndexOrDocValuesQuery(query, dvQuery); + DateMathParser parser = resolveDateMathParser(forcedDateParser, lowerTerm, upperTerm); + return dateRangeQuery( + lowerTerm, + upperTerm, + includeLower, + includeUpper, + timeZone, + parser, + context, + resolution, + name(), + (l, u) -> { + Query query; + if (isIndexed()) { + query = LongPoint.newRangeQuery(name(), l, u); + if (hasDocValues()) { + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + } else { + query = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); } - } else { - query = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); - } - if (hasDocValues() && context.indexSortedOnField(name())) { - query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); + if (hasDocValues() && context.indexSortedOnField(name())) { + query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); + } + return query; } - return query; - }); + ); } public static Query dateRangeQuery( @@ -785,6 +787,7 @@ public static Query dateRangeQuery( DateMathParser parser, SearchExecutionContext context, Resolution resolution, + String fieldName, BiFunction builder ) { return handleNow(context, nowSupplier -> { @@ -796,6 +799,9 @@ public static Query dateRangeQuery( if (includeLower == false) { ++l; } + if (fieldName.equals(DataStream.TIMESTAMP_FIELD_NAME)) { + context.setRangeTimestampFrom(l); + } } if (upperTerm == null) { u = Long.MAX_VALUE; @@ -951,6 +957,17 @@ public Relation isFieldWithinQuery( return isFieldWithinQuery(minValue, maxValue, from, to, includeLower, includeUpper, timeZone, dateParser, context); } + public DateMathParser resolveDateMathParser(DateMathParser dateParser, Object from, Object to) { + if (dateParser == null) { + if (from instanceof Number || to instanceof Number) { + // force epoch_millis + return EPOCH_MILLIS_PARSER; + } + return this.dateMathParser; + } + return dateParser; + } + public Relation isFieldWithinQuery( long minValue, long maxValue, @@ -962,14 +979,7 @@ public Relation isFieldWithinQuery( DateMathParser dateParser, QueryRewriteContext context ) { - if (dateParser == null) { - if (from instanceof Number || to instanceof Number) { - // force epoch_millis - dateParser = EPOCH_MILLIS_PARSER; - } else { - dateParser = this.dateMathParser; - } - } + dateParser = resolveDateMathParser(dateParser, from, to); long fromInclusive = Long.MIN_VALUE; if (from != null) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java index 7b744d8009e47..728b9cb6b1ae0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java @@ -297,6 +297,7 @@ public Query rangeQuery( parser, context, DateFieldMapper.Resolution.MILLISECONDS, + name(), (l, u) -> new LongScriptFieldRangeQuery(script, leafFactory(context)::newInstance, name(), l, u) ); } diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 93e356aa34f1d..fdad168019304 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -109,6 +109,8 @@ public class SearchExecutionContext extends QueryRewriteContext { private final Integer requestSize; private final MapperMetrics mapperMetrics; + private Long rangeTimestampFrom; + /** * Build a {@linkplain SearchExecutionContext}. */ @@ -236,6 +238,9 @@ public SearchExecutionContext(SearchExecutionContext source) { source.requestSize, source.mapperMetrics ); + // TODO address this + // setRangeEventIngestedFrom(source.rangeEventIngestedFrom); + // setRangeTimestampFrom(source.rangeTimestampFrom); } private SearchExecutionContext( @@ -300,6 +305,7 @@ private void reset() { this.lookup = null; this.namedQueries.clear(); this.nestedScope = new NestedScope(); + } // Set alias filter, so it can be applied for queries that need it (e.g. knn query) @@ -742,4 +748,22 @@ public void setRewriteToNamedQueries() { public boolean rewriteToNamedQuery() { return rewriteToNamedQueries; } + + /** + * Returns the minimum lower bound across the time ranges filters against the @timestamp field included in the query + */ + public Long getRangeTimestampFrom() { + return rangeTimestampFrom; + } + + /** + * Records the lower bound of a time range filter against the @timestamp field included in the query. For telemetry purposes. + */ + public void setRangeTimestampFrom(Long rangeTimestampFrom) { + if (this.rangeTimestampFrom == null) { + this.rangeTimestampFrom = rangeTimestampFrom; + } else { + this.rangeTimestampFrom = Math.min(rangeTimestampFrom, this.rangeTimestampFrom); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java index 38ec698af8d9c..ffd18d9712e3a 100644 --- a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java +++ b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java @@ -10,6 +10,7 @@ package org.elasticsearch.index.search.stats; import org.elasticsearch.action.search.SearchRequestAttributesExtractor; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.SearchOperationListener; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.ShardSearchRequest; @@ -42,16 +43,29 @@ public ShardSearchPhaseAPMMetrics(MeterRegistry meterRegistry) { @Override public void onQueryPhase(SearchContext searchContext, long tookInNanos) { - recordPhaseLatency(queryPhaseMetric, tookInNanos, searchContext.request()); + SearchExecutionContext searchExecutionContext = searchContext.getSearchExecutionContext(); + Long rangeTimestampFrom = searchExecutionContext.getRangeTimestampFrom(); + recordPhaseLatency(queryPhaseMetric, tookInNanos, searchContext.request(), rangeTimestampFrom); } @Override public void onFetchPhase(SearchContext searchContext, long tookInNanos) { - recordPhaseLatency(fetchPhaseMetric, tookInNanos, searchContext.request()); + SearchExecutionContext searchExecutionContext = searchContext.getSearchExecutionContext(); + Long rangeTimestampFrom = searchExecutionContext.getRangeTimestampFrom(); + recordPhaseLatency(fetchPhaseMetric, tookInNanos, searchContext.request(), rangeTimestampFrom); } - private static void recordPhaseLatency(LongHistogram histogramMetric, long tookInNanos, ShardSearchRequest request) { - Map attributes = SearchRequestAttributesExtractor.extractAttributes(request); + private static void recordPhaseLatency( + LongHistogram histogramMetric, + long tookInNanos, + ShardSearchRequest request, + Long rangeTimestampFrom + ) { + Map attributes = SearchRequestAttributesExtractor.extractAttributes( + request, + rangeTimestampFrom, + request.nowInMillis() + ); histogramMetric.record(TimeUnit.NANOSECONDS.toMillis(tookInNanos), attributes); } } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java index ad695eb49f99e..aee541cbcca35 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java @@ -361,4 +361,56 @@ public void testDepthLimit() { assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, false, false, null); } } + + public void testIntrospectTimeRange() { + long nowInMillis = System.currentTimeMillis(); + assertEquals("15_minutes", SearchRequestAttributesExtractor.introspectTimeRange(nowInMillis, nowInMillis)); + + long fifteenMinutesAgo = nowInMillis - (15 * 60 * 1000); + assertEquals( + "15_minutes", + SearchRequestAttributesExtractor.introspectTimeRange(randomLongBetween(fifteenMinutesAgo, nowInMillis), nowInMillis) + ); + + long oneHourAgo = nowInMillis - (60 * 60 * 1000); + assertEquals( + "1_hour", + SearchRequestAttributesExtractor.introspectTimeRange(randomLongBetween(oneHourAgo, fifteenMinutesAgo), nowInMillis) + ); + + long twelveHoursAgo = nowInMillis - (12 * 60 * 60 * 1000); + assertEquals( + "12_hours", + SearchRequestAttributesExtractor.introspectTimeRange(randomLongBetween(twelveHoursAgo, oneHourAgo), nowInMillis) + ); + + long oneDayAgo = nowInMillis - (24 * 60 * 60 * 1000); + assertEquals( + "1_day", + SearchRequestAttributesExtractor.introspectTimeRange(randomLongBetween(oneDayAgo, twelveHoursAgo), nowInMillis) + ); + + long threeDaysAgo = nowInMillis - (3 * 24 * 60 * 60 * 1000); + assertEquals( + "3_days", + SearchRequestAttributesExtractor.introspectTimeRange(randomLongBetween(threeDaysAgo, oneDayAgo), nowInMillis) + ); + + long sevenDaysAgo = nowInMillis - (7 * 24 * 60 * 60 * 1000); + assertEquals( + "7_days", + SearchRequestAttributesExtractor.introspectTimeRange(randomLongBetween(sevenDaysAgo, threeDaysAgo), nowInMillis) + ); + + long fourteenDaysAgo = nowInMillis - (14 * 24 * 60 * 60 * 1000); + assertEquals( + "14_days", + SearchRequestAttributesExtractor.introspectTimeRange(randomLongBetween(fourteenDaysAgo, sevenDaysAgo), nowInMillis) + ); + + assertEquals( + "older_than_14_days", + SearchRequestAttributesExtractor.introspectTimeRange(randomLongBetween(0, fourteenDaysAgo), nowInMillis) + ); + } } diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java index 865ea3d8bf6bd..25fa7d1f36b41 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java @@ -279,12 +279,13 @@ public void testTimeRangeFilterOneResult() { private static void assertTimeRangeAttributes(List measurements, String target, boolean isSystem) { for (Measurement measurement : measurements) { Map attributes = measurement.attributes(); - assertEquals(5, attributes.size()); + assertEquals(6, attributes.size()); assertEquals(target, attributes.get("target")); assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); assertEquals(true, attributes.get("range_timestamp")); assertEquals(isSystem, attributes.get(SearchRequestAttributesExtractor.SYSTEM_THREAD_ATTRIBUTE_NAME)); + assertEquals("older_than_14_days", attributes.get("timestamp_range_filter")); } } From 2f98053770bc17ddd19abaab50b50f4dd9d4943a Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 26 Sep 2025 12:10:13 +0200 Subject: [PATCH 2/3] iter --- .../org/elasticsearch/index/query/SearchExecutionContext.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index fdad168019304..1e8bd76ae03b8 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -238,9 +238,6 @@ public SearchExecutionContext(SearchExecutionContext source) { source.requestSize, source.mapperMetrics ); - // TODO address this - // setRangeEventIngestedFrom(source.rangeEventIngestedFrom); - // setRangeTimestampFrom(source.rangeTimestampFrom); } private SearchExecutionContext( From 4bebd1272d17a387e2e877fb0c7621d71bc58e90 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 26 Sep 2025 12:12:08 +0200 Subject: [PATCH 3/3] Update docs/changelog/135524.yaml --- docs/changelog/135524.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/135524.yaml diff --git a/docs/changelog/135524.yaml b/docs/changelog/135524.yaml new file mode 100644 index 0000000000000..e8b4eb5cc674c --- /dev/null +++ b/docs/changelog/135524.yaml @@ -0,0 +1,5 @@ +pr: 135524 +summary: Add time range bucketing attribute to APM shard search latency metrics +area: Search +type: enhancement +issues: []