diff --git a/docs/changelog/135549.yaml b/docs/changelog/135549.yaml new file mode 100644 index 0000000000000..c3567161b7eaf --- /dev/null +++ b/docs/changelog/135549.yaml @@ -0,0 +1,5 @@ +pr: 135549 +summary: Add time range bucketing attribute to APM took time latency metrics +area: Search +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumer.java b/server/src/main/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumer.java index a454181bf47a1..241141ea79e94 100644 --- a/server/src/main/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumer.java +++ b/server/src/main/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumer.java @@ -86,7 +86,8 @@ public SearchPhaseController.ReducedQueryPhase reduce() throws Exception { 1, 0, 0, - results.isEmpty() + results.isEmpty(), + null ); if (progressListener != SearchProgressListener.NOOP) { progressListener.notifyFinalReduce( diff --git a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java index 25238711c5c1c..07b183629fcb5 100644 --- a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java @@ -251,7 +251,8 @@ private SearchPhaseController.ReducedQueryPhase newReducedQueryPhaseResults( reducedQueryPhase.numReducePhases(), reducedQueryPhase.size(), reducedQueryPhase.from(), - reducedQueryPhase.isEmptyResult() + reducedQueryPhase.isEmptyResult(), + reducedQueryPhase.timeRangeFilterFromMillis() ); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 2df4c05722908..774bb4c20faa6 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -487,7 +487,8 @@ static ReducedQueryPhase reducedQueryPhase( numReducePhases, 0, 0, - true + true, + null ); } final List nonNullResults = new ArrayList<>(); @@ -516,6 +517,7 @@ static ReducedQueryPhase reducedQueryPhase( : Collections.emptyMap(); int from = 0; int size = 0; + Long timeRangeFilterFromMillis = null; DocValueFormat[] sortValueFormats = null; for (QuerySearchResult result : nonNullResults) { from = result.from(); @@ -525,6 +527,16 @@ static ReducedQueryPhase reducedQueryPhase( sortValueFormats = result.sortValueFormats(); } + if (result.getTimeRangeFilterFromMillis() != null) { + if (timeRangeFilterFromMillis == null) { + timeRangeFilterFromMillis = result.getTimeRangeFilterFromMillis(); + } else { + // all shards should hold the same value, besides edge cases like different mappings + // for event.ingested and @timestamp across indices being searched + timeRangeFilterFromMillis = Math.min(result.getTimeRangeFilterFromMillis(), timeRangeFilterFromMillis); + } + } + if (hasSuggest) { assert result.suggest() != null; for (Suggestion> suggestion : result.suggest()) { @@ -579,7 +591,8 @@ static ReducedQueryPhase reducedQueryPhase( numReducePhases, size, from, - false + false, + timeRangeFilterFromMillis ); } @@ -662,7 +675,8 @@ public record ReducedQueryPhase( // the offset into the merged top hits int from, // true iff the query phase had no results. Otherwise false - boolean isEmptyResult + boolean isEmptyResult, + Long timeRangeFilterFromMillis ) { public ReducedQueryPhase { @@ -683,7 +697,8 @@ public SearchResponseSections buildResponse(SearchHits hits, Collection extractAttributes(SearchRequest searchRequest, /** * Introspects the provided shard search request and extracts metadata from it about some of its characteristics. */ - public static Map extractAttributes(ShardSearchRequest shardSearchRequest, Long rangeTimestampFrom, long nowInMillis) { + public static Map extractAttributes( + ShardSearchRequest shardSearchRequest, + Long timeRangeFilterFromMillis, + long nowInMillis + ) { Map attributes = extractAttributes( shardSearchRequest.source(), shardSearchRequest.scroll(), - rangeTimestampFrom, + timeRangeFilterFromMillis, nowInMillis, shardSearchRequest.shardId().getIndexName() ); @@ -69,7 +75,7 @@ public static Map extractAttributes(ShardSearchRequest shardSear private static Map extractAttributes( SearchSourceBuilder searchSourceBuilder, TimeValue scroll, - Long rangeTimestampFrom, + Long timeRangeFilterFromMillis, long nowInMillis, String... localIndices ) { @@ -107,9 +113,9 @@ private static Map extractAttributes( } final boolean hasKnn = searchSourceBuilder.knnSearch().isEmpty() == false || queryMetadataBuilder.knnQuery; - String timestampRangeFilter = null; - if (rangeTimestampFrom != null) { - timestampRangeFilter = introspectTimeRange(rangeTimestampFrom, nowInMillis); + String timeRangeFilterFrom = null; + if (timeRangeFilterFromMillis != null) { + timeRangeFilterFrom = introspectTimeRange(timeRangeFilterFromMillis, nowInMillis); } return buildAttributesMap( target, @@ -119,7 +125,7 @@ private static Map extractAttributes( queryMetadataBuilder.rangeOnTimestamp, queryMetadataBuilder.rangeOnEventIngested, pitOrScroll, - timestampRangeFilter + timeRangeFilterFrom ); } @@ -131,7 +137,7 @@ private static Map buildAttributesMap( boolean rangeOnTimestamp, boolean rangeOnEventIngested, String pitOrScroll, - String timestampRangeFilter + String timeRangeFilterFrom ) { Map attributes = new HashMap<>(5, 1.0f); attributes.put(TARGET_ATTRIBUTE, target); @@ -143,14 +149,18 @@ private static Map buildAttributesMap( if (knn) { attributes.put(KNN_ATTRIBUTE, knn); } - if (rangeOnTimestamp) { - attributes.put(RANGE_TIMESTAMP_ATTRIBUTE, rangeOnTimestamp); - } - if (rangeOnEventIngested) { - attributes.put(RANGE_EVENT_INGESTED_ATTRIBUTE, rangeOnEventIngested); + if (rangeOnTimestamp && rangeOnEventIngested) { + attributes.put( + TIME_RANGE_FILTER_FIELD_ATTRIBUTE, + DataStream.TIMESTAMP_FIELD_NAME + "_AND_" + IndexMetadata.EVENT_INGESTED_FIELD_NAME + ); + } else if (rangeOnEventIngested) { + attributes.put(TIME_RANGE_FILTER_FIELD_ATTRIBUTE, IndexMetadata.EVENT_INGESTED_FIELD_NAME); + } else if (rangeOnTimestamp) { + attributes.put(TIME_RANGE_FILTER_FIELD_ATTRIBUTE, DataStream.TIMESTAMP_FIELD_NAME); } - if (timestampRangeFilter != null) { - attributes.put(TIMESTAMP_RANGE_FILTER_ATTRIBUTE, timestampRangeFilter); + if (timeRangeFilterFrom != null) { + attributes.put(TIME_RANGE_FILTER_FROM_ATTRIBUTE, timeRangeFilterFrom); } return attributes; } @@ -166,9 +176,8 @@ private static final class QueryMetadataBuilder { static final String QUERY_TYPE_ATTRIBUTE = "query_type"; static final String PIT_SCROLL_ATTRIBUTE = "pit_scroll"; static final String KNN_ATTRIBUTE = "knn"; - static final String RANGE_TIMESTAMP_ATTRIBUTE = "range_timestamp"; - static final String RANGE_EVENT_INGESTED_ATTRIBUTE = "range_event_ingested"; - static final String TIMESTAMP_RANGE_FILTER_ATTRIBUTE = "timestamp_range_filter"; + static final String TIME_RANGE_FILTER_FIELD_ATTRIBUTE = "time_range_filter_field"; + static final String TIME_RANGE_FILTER_FROM_ATTRIBUTE = "time_range_filter_from"; private static final String TARGET_KIBANA = ".kibana"; private static final String TARGET_ML = ".ml"; @@ -303,6 +312,10 @@ private static void introspectQueryBuilder(QueryBuilder queryBuilder, QueryMetad introspectQueryBuilder(nested.query(), queryMetadataBuilder, ++level); break; case RangeQueryBuilder range: + // Note that the outcome of this switch differs depending on whether it is executed on the coord node, or data node. + // Data nodes perform query rewrite on each shard. That means that a query that reports a certain time range filter at the + // coordinator, may not report the same for all the shards it targets, but rather only for those that do end up executing + // a true range query at the shard level. switch (range.fieldName()) { // don't track unbounded ranges, they translate to either match_none if the field does not exist // or match_all if the field is mapped @@ -343,9 +356,16 @@ private enum TimeRangeBucket { } } - static String introspectTimeRange(long timeRangeFrom, long nowInMillis) { + public static void addTimeRangeAttribute(Long timeRangeFrom, long nowInMillis, Map attributes) { + if (timeRangeFrom != null) { + String timestampRangeFilter = introspectTimeRange(timeRangeFrom, nowInMillis); + attributes.put(TIME_RANGE_FILTER_FROM_ATTRIBUTE, timestampRangeFilter); + } + } + + static String introspectTimeRange(long timeRangeFromMillis, long nowInMillis) { for (TimeRangeBucket value : TimeRangeBucket.values()) { - if (timeRangeFrom >= nowInMillis - value.millis) { + if (timeRangeFromMillis >= nowInMillis - value.millis) { return value.bucketName; } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java index 7b82116e5b447..3b67d0b5ac160 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java @@ -87,6 +87,8 @@ public class SearchResponse extends ActionResponse implements ChunkedToXContentO private final ShardSearchFailure[] shardFailures; private final Clusters clusters; private final long tookInMillis; + // only used for telemetry purposes on the coordinating node, where the search response gets created + private transient Long timeRangeFilterFromMillis; private final RefCounted refCounted = LeakTracker.wrap(new SimpleRefCounted()); @@ -187,6 +189,7 @@ public SearchResponse( clusters, pointInTimeId ); + this.timeRangeFilterFromMillis = searchResponseSections.timeRangeFilterFromMillis; } public SearchResponse( @@ -464,6 +467,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalBytesReference(pointInTimeId); } + public Long getTimeRangeFilterFromMillis() { + return timeRangeFilterFromMillis; + } + @Override public String toString() { return hasReferences() == false ? "SearchResponse[released]" : Strings.toString(this); diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java index c63fcf5c3c025..c13578d7aeb9b 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java @@ -32,7 +32,8 @@ public class SearchResponseSections implements Releasable { false, null, null, - 1 + 1, + null ); public static final SearchResponseSections EMPTY_WITHOUT_TOTAL_HITS = new SearchResponseSections( SearchHits.EMPTY_WITHOUT_TOTAL_HITS, @@ -41,7 +42,8 @@ public class SearchResponseSections implements Releasable { false, null, null, - 1 + 1, + null ); protected final SearchHits hits; protected final InternalAggregations aggregations; @@ -50,6 +52,7 @@ public class SearchResponseSections implements Releasable { protected final boolean timedOut; protected final Boolean terminatedEarly; protected final int numReducePhases; + protected final Long timeRangeFilterFromMillis; public SearchResponseSections( SearchHits hits, @@ -58,7 +61,8 @@ public SearchResponseSections( boolean timedOut, Boolean terminatedEarly, SearchProfileResults profileResults, - int numReducePhases + int numReducePhases, + Long timeRangeFilterFromMillis ) { this.hits = hits; this.aggregations = aggregations; @@ -67,6 +71,7 @@ public SearchResponseSections( this.timedOut = timedOut; this.terminatedEarly = terminatedEarly; this.numReducePhases = numReducePhases; + this.timeRangeFilterFromMillis = timeRangeFilterFromMillis; } public final SearchHits hits() { diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 3385f53c9ed51..16389f6137f81 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -430,7 +430,12 @@ public void onFailure(Exception e) { Arrays.stream(resolvedIndices.getConcreteLocalIndices()).map(Index::getName).toArray(String[]::new) ); if (collectCCSTelemetry == false || resolvedIndices.getRemoteClusterIndices().isEmpty()) { - searchResponseActionListener = new SearchTelemetryListener(delegate, searchResponseMetrics, searchRequestAttributes); + searchResponseActionListener = new SearchTelemetryListener( + delegate, + searchResponseMetrics, + searchRequestAttributes, + timeProvider.absoluteStartMillis() + ); } else { CCSUsage.Builder usageBuilder = new CCSUsage.Builder(); usageBuilder.setRemotesCount(resolvedIndices.getRemoteClusterIndices().size()); @@ -459,6 +464,7 @@ public void onFailure(Exception e) { delegate, searchResponseMetrics, searchRequestAttributes, + timeProvider.absoluteStartMillis(), usageService, usageBuilder ); @@ -2046,6 +2052,7 @@ static String[] ignoreBlockedIndices(ProjectState projectState, String[] concret private static class SearchTelemetryListener extends DelegatingActionListener { private final CCSUsage.Builder usageBuilder; private final SearchResponseMetrics searchResponseMetrics; + private final long nowInMillis; private final UsageService usageService; private final boolean collectCCSTelemetry; private final Map searchRequestAttributes; @@ -2054,12 +2061,14 @@ private static class SearchTelemetryListener extends DelegatingActionListener listener, SearchResponseMetrics searchResponseMetrics, Map searchRequestAttributes, + long nowInMillis, UsageService usageService, CCSUsage.Builder usageBuilder ) { super(listener); this.searchResponseMetrics = searchResponseMetrics; this.searchRequestAttributes = searchRequestAttributes; + this.nowInMillis = nowInMillis; this.collectCCSTelemetry = true; this.usageService = usageService; this.usageBuilder = usageBuilder; @@ -2068,11 +2077,13 @@ private static class SearchTelemetryListener extends DelegatingActionListener listener, SearchResponseMetrics searchResponseMetrics, - Map searchRequestAttributes + Map searchRequestAttributes, + long nowInMillis ) { super(listener); this.searchResponseMetrics = searchResponseMetrics; this.searchRequestAttributes = searchRequestAttributes; + this.nowInMillis = nowInMillis; this.collectCCSTelemetry = false; this.usageService = null; this.usageBuilder = null; @@ -2081,7 +2092,12 @@ private static class SearchTelemetryListener extends DelegatingActionListener 0) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 859e60e628547..030c202b08080 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -25,7 +25,6 @@ import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.Query; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; @@ -804,9 +803,7 @@ public static Query dateRangeQuery( if (includeLower == false) { ++l; } - if (fieldName.equals(DataStream.TIMESTAMP_FIELD_NAME)) { - context.setRangeTimestampFrom(l); - } + context.setTimeRangeFilterFromMillis(fieldName, l, resolution); } if (upperTerm == null) { u = Long.MAX_VALUE; @@ -949,7 +946,7 @@ public Relation isFieldWithinQuery( minValue = Long.min(minValue, skipper.minValue()); maxValue = Long.max(maxValue, skipper.maxValue()); } - return isFieldWithinQuery(minValue, maxValue, from, to, includeLower, includeUpper, timeZone, dateParser, context); + return isFieldWithinQuery(minValue, maxValue, from, to, includeLower, includeUpper, timeZone, dateParser, context, name()); } byte[] minPackedValue = PointValues.getMinPackedValue(reader, name()); if (minPackedValue == null) { @@ -959,7 +956,7 @@ public Relation isFieldWithinQuery( long minValue = LongPoint.decodeDimension(minPackedValue, 0); long maxValue = LongPoint.decodeDimension(PointValues.getMaxPackedValue(reader, name()), 0); - return isFieldWithinQuery(minValue, maxValue, from, to, includeLower, includeUpper, timeZone, dateParser, context); + return isFieldWithinQuery(minValue, maxValue, from, to, includeLower, includeUpper, timeZone, dateParser, context, name()); } public DateMathParser resolveDateMathParser(DateMathParser dateParser, Object from, Object to) { @@ -982,7 +979,8 @@ public Relation isFieldWithinQuery( boolean includeUpper, ZoneId timeZone, DateMathParser dateParser, - QueryRewriteContext context + QueryRewriteContext context, + String fieldName ) { dateParser = resolveDateMathParser(dateParser, from, to); @@ -995,6 +993,9 @@ public Relation isFieldWithinQuery( } ++fromInclusive; } + // we set the time range filter from during rewrite, because this may be the only time we ever parse it, + // in case the shard if filtered out and does not run the query phase or all its docs are within the bounds. + context.setTimeRangeFilterFromMillis(fieldName, fromInclusive, resolution); } long toInclusive = Long.MAX_VALUE; diff --git a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java index 1225a070a7c00..5944fc3d8df7a 100644 --- a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java @@ -300,8 +300,14 @@ public String getWriteableName() { protected Query doToQuery(SearchExecutionContext context) throws IOException { BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); addBooleanClauses(context, booleanQueryBuilder, mustClauses, BooleanClause.Occur.MUST); - addBooleanClauses(context, booleanQueryBuilder, mustNotClauses, BooleanClause.Occur.MUST_NOT); - addBooleanClauses(context, booleanQueryBuilder, shouldClauses, BooleanClause.Occur.SHOULD); + try { + // disable tracking of the @timestamp range for must_not and should clauses + context.setTrackTimeRangeFilterFrom(false); + addBooleanClauses(context, booleanQueryBuilder, mustNotClauses, BooleanClause.Occur.MUST_NOT); + addBooleanClauses(context, booleanQueryBuilder, shouldClauses, BooleanClause.Occur.SHOULD); + } finally { + context.setTrackTimeRangeFilterFrom(true); + } addBooleanClauses(context, booleanQueryBuilder, filterClauses, BooleanClause.Occur.FILTER); BooleanQuery booleanQuery = booleanQueryBuilder.build(); if (booleanQuery.clauses().isEmpty()) { @@ -348,9 +354,23 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws return new MatchAllQueryBuilder().boost(boost()).queryName(queryName()); } changed |= rewriteClauses(queryRewriteContext, mustClauses, newBuilder::must); - changed |= rewriteClauses(queryRewriteContext, mustNotClauses, newBuilder::mustNot); + + try { + // disable tracking of the @timestamp range for must_not clauses + queryRewriteContext.setTrackTimeRangeFilterFrom(false); + changed |= rewriteClauses(queryRewriteContext, mustNotClauses, newBuilder::mustNot); + } finally { + queryRewriteContext.setTrackTimeRangeFilterFrom(true); + } changed |= rewriteClauses(queryRewriteContext, filterClauses, newBuilder::filter); - changed |= rewriteClauses(queryRewriteContext, shouldClauses, newBuilder::should); + try { + // disable tracking of the @timestamp range for should clauses + queryRewriteContext.setTrackTimeRangeFilterFrom(false); + changed |= rewriteClauses(queryRewriteContext, shouldClauses, newBuilder::should); + } finally { + queryRewriteContext.setTrackTimeRangeFilterFrom(true); + } + // early termination when must clause is empty and optional clauses is returning MatchNoneQueryBuilder if (mustClauses.size() == 0 && filterClauses.size() == 0 diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java index 738ad19758f91..5ca77374dee59 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java @@ -12,6 +12,8 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ResolvedIndices; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Iterators; @@ -23,6 +25,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperService; @@ -77,6 +80,8 @@ public class QueryRewriteContext { private QueryRewriteInterceptor queryRewriteInterceptor; private final Boolean ccsMinimizeRoundTrips; private final boolean isExplain; + private Long timeRangeFilterFromMillis; + private boolean trackTimeRangeFilterFrom = true; public QueryRewriteContext( final XContentParserConfiguration parserConfiguration, @@ -515,4 +520,48 @@ public void setQueryRewriteInterceptor(QueryRewriteInterceptor queryRewriteInter this.queryRewriteInterceptor = queryRewriteInterceptor; } + /** + * Returns the minimum lower bound across the time ranges filters against the @timestamp field included in the query + */ + public Long getTimeRangeFilterFromMillis() { + return timeRangeFilterFromMillis; + } + + /** + * Optionally records the lower bound of a time range filter included in the query. For telemetry purposes. + */ + public void setTimeRangeFilterFromMillis(String fieldName, long timeRangeFilterFromMillis, DateFieldMapper.Resolution resolution) { + if (trackTimeRangeFilterFrom) { + if (DataStream.TIMESTAMP_FIELD_NAME.equals(fieldName) || IndexMetadata.EVENT_INGESTED_FIELD_NAME.equals(fieldName)) { + // if we got a timestamp with nanoseconds precision, round it down to millis + if (resolution == DateFieldMapper.Resolution.NANOSECONDS) { + timeRangeFilterFromMillis = timeRangeFilterFromMillis / 1_000_000; + } + if (this.timeRangeFilterFromMillis == null) { + this.timeRangeFilterFromMillis = timeRangeFilterFromMillis; + } else { + // if there's more range filters on timestamp, we'll take the lowest of the lower bounds + this.timeRangeFilterFromMillis = Math.min(timeRangeFilterFromMillis, this.timeRangeFilterFromMillis); + } + } + } + } + + /** + * Records the lower bound of a time range filter included in the query. For telemetry purposes. + * Similar to {@link #setTimeRangeFilterFromMillis(String, long, DateFieldMapper.Resolution)} but used to copy the value from + * another instance of the context, that had its value previously set. + */ + public void setTimeRangeFilterFromMillis(long timeRangeFilterFromMillis) { + this.timeRangeFilterFromMillis = timeRangeFilterFromMillis; + } + + /** + * Enables or disables the tracking of the lower bound for time range filters against the @timestamp field, + * done via {@link #setTimeRangeFilterFromMillis(String, long, DateFieldMapper.Resolution)}. Tracking is enabled by default, + * and explicitly disabled to ensure we don't record the bound for range queries within should and must_not clauses. + */ + public void setTrackTimeRangeFilterFrom(boolean trackTimeRangeFilterFrom) { + this.trackTimeRangeFilterFrom = trackTimeRangeFilterFrom; + } } diff --git a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java index c1787cf0a84cc..95b117bcbc311 100644 --- a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java @@ -448,7 +448,8 @@ protected MappedFieldType.Relation getRelation(final CoordinatorRewriteContext c includeUpper, timeZone, dateMathParser, - coordinatorRewriteContext + coordinatorRewriteContext, + dateFieldType.name() ); } // If the field type is null or not of type DataFieldType then we have no idea whether this range query will match during diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 1e8bd76ae03b8..6919bd725133c 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -109,8 +109,6 @@ public class SearchExecutionContext extends QueryRewriteContext { private final Integer requestSize; private final MapperMetrics mapperMetrics; - private Long rangeTimestampFrom; - /** * Build a {@linkplain SearchExecutionContext}. */ @@ -745,22 +743,4 @@ public void setRewriteToNamedQueries() { public boolean rewriteToNamedQuery() { return rewriteToNamedQueries; } - - /** - * Returns the minimum lower bound across the time ranges filters against the @timestamp field included in the query - */ - public Long getRangeTimestampFrom() { - return rangeTimestampFrom; - } - - /** - * Records the lower bound of a time range filter against the @timestamp field included in the query. For telemetry purposes. - */ - public void setRangeTimestampFrom(Long rangeTimestampFrom) { - if (this.rangeTimestampFrom == null) { - this.rangeTimestampFrom = rangeTimestampFrom; - } else { - this.rangeTimestampFrom = Math.min(rangeTimestampFrom, this.rangeTimestampFrom); - } - } } diff --git a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java index fc823f4c51f7c..f35be08dc6472 100644 --- a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java +++ b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java @@ -56,15 +56,15 @@ public void onDfsPhase(SearchContext searchContext, long tookInNanos) { @Override public void onQueryPhase(SearchContext searchContext, long tookInNanos) { SearchExecutionContext searchExecutionContext = searchContext.getSearchExecutionContext(); - Long rangeTimestampFrom = searchExecutionContext.getRangeTimestampFrom(); - recordPhaseLatency(queryPhaseMetric, tookInNanos, searchContext.request(), rangeTimestampFrom); + Long timeRangeFilterFromMillis = searchExecutionContext.getTimeRangeFilterFromMillis(); + recordPhaseLatency(queryPhaseMetric, tookInNanos, searchContext.request(), timeRangeFilterFromMillis); } @Override public void onFetchPhase(SearchContext searchContext, long tookInNanos) { SearchExecutionContext searchExecutionContext = searchContext.getSearchExecutionContext(); - Long rangeTimestampFrom = searchExecutionContext.getRangeTimestampFrom(); - recordPhaseLatency(fetchPhaseMetric, tookInNanos, searchContext.request(), rangeTimestampFrom); + Long timeRangeFilterFromMillis = searchExecutionContext.getTimeRangeFilterFromMillis(); + recordPhaseLatency(fetchPhaseMetric, tookInNanos, searchContext.request(), timeRangeFilterFromMillis); } private static void recordPhaseLatency(LongHistogram histogramMetric, long tookInNanos) { @@ -75,11 +75,11 @@ private static void recordPhaseLatency( LongHistogram histogramMetric, long tookInNanos, ShardSearchRequest request, - Long rangeTimestampFrom + Long timeRangeFilterFromMillis ) { Map attributes = SearchRequestAttributesExtractor.extractAttributes( request, - rangeTimestampFrom, + timeRangeFilterFromMillis, request.nowInMillis() ); histogramMetric.record(TimeUnit.NANOSECONDS.toMillis(tookInNanos), attributes); diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java index 900ec9369e361..ae9f14de4c24d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java @@ -73,7 +73,8 @@ public long recordTookTimeForSearchScroll(long tookTime) { return tookTime; } - public long recordTookTime(long tookTime, Map attributes) { + public long recordTookTime(long tookTime, Long timeRangeFilterFromMillis, long nowInMillis, Map attributes) { + SearchRequestAttributesExtractor.addTimeRangeAttribute(timeRangeFilterFromMillis, nowInMillis, attributes); tookDurationTotalMillisHistogram.record(tookTime, attributes); return tookTime; } diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 4c26be97da1b6..2c53df81041b2 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -1442,6 +1442,11 @@ private DefaultSearchContext createSearchContext( // during rewrite and normalized / evaluate templates etc. SearchExecutionContext context = new SearchExecutionContext(searchContext.getSearchExecutionContext()); Rewriteable.rewrite(request.getRewriteable(), context, true); + if (context.getTimeRangeFilterFromMillis() != null) { + // range queries may get rewritten to match_all or a range with open bounds. Rewriting in that case is the only place + // where we parse the date and set it to the context. We need to propagate it back from the clone into the original context + searchContext.getSearchExecutionContext().setTimeRangeFilterFromMillis(context.getTimeRangeFilterFromMillis()); + } assert searchContext.getSearchExecutionContext().isCacheable(); success = true; } finally { diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 5fcfb2b9766cd..7736c05b89728 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -104,7 +104,7 @@ static void executeRank(SearchContext searchContext) throws QueryPhaseExecutionE queryPhaseRankShardContext.rankWindowSize() ) ) { - QueryPhase.addCollectorsAndSearch(rankSearchContext); + QueryPhase.addCollectorsAndSearch(rankSearchContext, null); QuerySearchResult rrfQuerySearchResult = rankSearchContext.queryResult(); rrfRankResults.add(rrfQuerySearchResult.topDocs().topDocs); serviceTimeEWMA += rrfQuerySearchResult.serviceTimeEWMA(); @@ -140,7 +140,7 @@ static void executeQuery(SearchContext searchContext) throws QueryPhaseExecution // here to make sure it happens during the QUERY phase AggregationPhase.preProcess(searchContext); - addCollectorsAndSearch(searchContext); + addCollectorsAndSearch(searchContext, searchContext.getSearchExecutionContext().getTimeRangeFilterFromMillis()); RescorePhase.execute(searchContext); SuggestPhase.execute(searchContext); @@ -153,10 +153,11 @@ static void executeQuery(SearchContext searchContext) throws QueryPhaseExecution * In a package-private method so that it can be tested without having to * wire everything (mapperService, etc.) */ - static void addCollectorsAndSearch(SearchContext searchContext) throws QueryPhaseExecutionException { + static void addCollectorsAndSearch(SearchContext searchContext, Long timeRangeFilterFromMillis) throws QueryPhaseExecutionException { final ContextIndexSearcher searcher = searchContext.searcher(); final IndexReader reader = searcher.getIndexReader(); QuerySearchResult queryResult = searchContext.queryResult(); + queryResult.setTimeRangeFilterFromMillis(timeRangeFilterFromMillis); queryResult.searchTimedOut(false); try { queryResult.from(searchContext.from()); diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java index d11457ca91536..1b3af6c22ca51 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java @@ -44,7 +44,7 @@ import static org.elasticsearch.common.lucene.Lucene.writeTopDocs; public final class QuerySearchResult extends SearchPhaseResult { - + private static final TransportVersion TIMESTAMP_RANGE_TELEMETRY = TransportVersion.fromName("timestamp_range_telemetry"); private static final TransportVersion BATCHED_QUERY_PHASE_VERSION = TransportVersion.fromName("batched_query_phase_version"); private int from; @@ -80,6 +80,9 @@ public final class QuerySearchResult extends SearchPhaseResult { private final SubscribableListener aggsContextReleased; + @Nullable + private Long timeRangeFilterFromMillis; + public QuerySearchResult() { this(false); } @@ -456,6 +459,9 @@ private void readFromWithId(ShardSearchContextId id, StreamInput in, boolean del reduced = in.readBoolean(); } } + if (in.getTransportVersion().supports(TIMESTAMP_RANGE_TELEMETRY)) { + timeRangeFilterFromMillis = in.readOptionalLong(); + } success = true; } finally { if (success == false) { @@ -526,6 +532,9 @@ public void writeToNoId(StreamOutput out) throws IOException { if (versionSupportsBatchedExecution(out.getTransportVersion())) { out.writeBoolean(reduced); } + if (out.getTransportVersion().supports(TIMESTAMP_RANGE_TELEMETRY)) { + out.writeOptionalLong(timeRangeFilterFromMillis); + } } @Nullable @@ -577,4 +586,12 @@ public boolean hasReferences() { private static boolean versionSupportsBatchedExecution(TransportVersion transportVersion) { return transportVersion.supports(BATCHED_QUERY_PHASE_VERSION); } + + public Long getTimeRangeFilterFromMillis() { + return timeRangeFilterFromMillis; + } + + public void setTimeRangeFilterFromMillis(Long timeRangeFilterFromMillis) { + this.timeRangeFilterFromMillis = timeRangeFilterFromMillis; + } } diff --git a/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv b/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv new file mode 100644 index 0000000000000..c6a65950bd0ab --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv @@ -0,0 +1 @@ +9188000 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index a1daf1f9747d4..fe8be2e1f55a8 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -resolved_index_expressions,9187000 +timestamp_range_telemetry,9188000 diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index a08dbc80e6b9c..a1353bc3ebcc3 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -99,7 +99,16 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL List mSearchResponses = new ArrayList<>(numInnerHits); for (int innerHitNum = 0; innerHitNum < numInnerHits; innerHitNum++) { try ( - var sections = new SearchResponseSections(collapsedHits.get(innerHitNum), null, null, false, null, null, 1) + var sections = new SearchResponseSections( + collapsedHits.get(innerHitNum), + null, + null, + false, + null, + null, + 1, + null + ) ) { mockSearchPhaseContext.sendSearchResponse(sections, null); } @@ -126,7 +135,8 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL false, null, null, - 1 + 1, + null ), null ); @@ -199,7 +209,8 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL false, null, null, - 1 + 1, + null ) ) { ExpandSearchPhase phase = newExpandSearchPhase(mockSearchPhaseContext, searchResponseSections, null); @@ -237,7 +248,8 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL false, null, null, - 1 + 1, + null ), null ); @@ -270,7 +282,7 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL ); SearchHits hits = SearchHits.empty(new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0f); - final SearchResponseSections searchResponseSections = new SearchResponseSections(hits, null, null, false, null, null, 1); + final SearchResponseSections searchResponseSections = new SearchResponseSections(hits, null, null, false, null, null, 1, null); ExpandSearchPhase phase = newExpandSearchPhase(mockSearchPhaseContext, searchResponseSections, null); phase.run(); mockSearchPhaseContext.assertNoFailure(); @@ -322,7 +334,8 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL false, null, null, - 1 + 1, + null ) ) { ExpandSearchPhase phase = newExpandSearchPhase(mockSearchPhaseContext, searchResponseSections, null); @@ -393,7 +406,8 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL false, null, null, - 1 + 1, + null ) ) { ExpandSearchPhase phase = newExpandSearchPhase(mockSearchPhaseContext, searchResponseSections, new AtomicArray<>(0)); diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchLookupFieldsPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchLookupFieldsPhaseTests.java index b03d2be165937..fe4259b582528 100644 --- a/server/src/test/java/org/elasticsearch/action/search/FetchLookupFieldsPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/FetchLookupFieldsPhaseTests.java @@ -54,7 +54,8 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL false, null, null, - 1 + 1, + null ) ) { FetchLookupFieldsPhase phase = new FetchLookupFieldsPhase(searchPhaseContext, sections, null); @@ -193,7 +194,8 @@ void sendExecuteMultiSearch( false, null, null, - 1 + 1, + null ) ) { FetchLookupFieldsPhase phase = new FetchLookupFieldsPhase(searchPhaseContext, sections, null); diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java index aee541cbcca35..cbb684acb39bf 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java @@ -137,15 +137,17 @@ private static void assertAttributes( } else { assertNull(attributes.get(SearchRequestAttributesExtractor.KNN_ATTRIBUTE)); } - if (rangeOnTimestamp) { - assertEquals(rangeOnTimestamp, attributes.get(SearchRequestAttributesExtractor.RANGE_TIMESTAMP_ATTRIBUTE)); - } else { - assertNull(attributes.get(SearchRequestAttributesExtractor.RANGE_TIMESTAMP_ATTRIBUTE)); - } - if (rangeOnEventIngested) { - assertEquals(rangeOnEventIngested, attributes.get(SearchRequestAttributesExtractor.RANGE_EVENT_INGESTED_ATTRIBUTE)); + if (rangeOnTimestamp && rangeOnEventIngested) { + assertEquals( + "@timestamp_AND_event.ingested", + attributes.get(SearchRequestAttributesExtractor.TIME_RANGE_FILTER_FIELD_ATTRIBUTE) + ); + } else if (rangeOnTimestamp) { + assertEquals("@timestamp", attributes.get(SearchRequestAttributesExtractor.TIME_RANGE_FILTER_FIELD_ATTRIBUTE)); + } else if (rangeOnEventIngested) { + assertEquals("event.ingested", attributes.get(SearchRequestAttributesExtractor.TIME_RANGE_FILTER_FIELD_ATTRIBUTE)); } else { - assertNull(attributes.get(SearchRequestAttributesExtractor.RANGE_EVENT_INGESTED_ATTRIBUTE)); + assertNull(attributes.get(SearchRequestAttributesExtractor.TIME_RANGE_FILTER_FIELD_ATTRIBUTE)); } } diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java index 792b676f6002b..4323541a9c8af 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.search.rescore.QueryRescorerBuilder; import org.elasticsearch.search.retriever.RescorerRetrieverBuilder; import org.elasticsearch.search.retriever.StandardRetrieverBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.telemetry.Measurement; import org.elasticsearch.telemetry.TestTelemetryPlugin; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -36,10 +37,14 @@ import org.junit.After; import org.junit.Before; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.Arrays; import java.util.Collection; import java.util.Comparator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -52,11 +57,11 @@ public class SearchTookTimeTelemetryTests extends ESSingleNodeTestCase { private static final String indexName = "test_search_metrics2"; - - @Override - protected boolean resetNodeAfterTest() { - return true; - } + private static final String indexNameNanoPrecision = "nano_search_metrics2"; + private static final String singleShardIndexName = "single_shard_test_search_metric"; + private static final LocalDateTime NOW = LocalDateTime.now(ZoneOffset.UTC); + private static final DateTimeFormatter FORMATTER_MILLIS = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT); + private static final DateTimeFormatter FORMATTER_NANOS = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.nnnnnnnnn", Locale.ROOT); @Before public void setUpIndex() { @@ -69,8 +74,95 @@ public void setUpIndex() { .build() ); ensureGreen(indexName); - prepareIndex(indexName).setId("1").setSource("body", "foo", "@timestamp", "2024-11-01").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex(indexName).setId("2").setSource("body", "foo", "@timestamp", "2024-12-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(indexName).setId("1") + .setSource("body", "foo", "@timestamp", "2024-11-01", "event.ingested", "2024-11-01") + .setRefreshPolicy(IMMEDIATE) + .get(); + prepareIndex(indexName).setId("2") + .setSource("body", "foo", "@timestamp", "2024-12-01", "event.ingested", "2024-12-01") + .setRefreshPolicy(IMMEDIATE) + .get(); + + // we use a single shard index to test the case where query and fetch execute in the same round-trip + createIndex( + singleShardIndexName, + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build() + ); + ensureGreen(singleShardIndexName); + prepareIndex(singleShardIndexName).setId("1") + .setSource("body", "foo", "@timestamp", NOW.minusMinutes(5).withSecond(randomIntBetween(0, 59)).format(FORMATTER_MILLIS)) + .setRefreshPolicy(IMMEDIATE) + .get(); + prepareIndex(singleShardIndexName).setId("2") + .setSource("body", "foo", "@timestamp", NOW.minusMinutes(30).withSecond(randomIntBetween(0, 59)).format(FORMATTER_MILLIS)) + .setRefreshPolicy(IMMEDIATE) + .get(); + + createIndex( + indexNameNanoPrecision, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, num_primaries) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build(), + "_doc", + "@timestamp", + "type=date_nanos" + ); + ensureGreen(indexNameNanoPrecision); + prepareIndex(indexNameNanoPrecision).setId("10") + .setSource( + "body", + "foo", + "@timestamp", + NOW.minusMinutes(2).withNano(randomIntBetween(0, 1_000_000_000)).format(FORMATTER_NANOS) + ) + .setRefreshPolicy(IMMEDIATE) + .get(); + prepareIndex(indexNameNanoPrecision).setId("11") + .setSource( + "body", + "foo", + "@timestamp", + NOW.minusMinutes(3).withNano(randomIntBetween(0, 1_000_000_000)).format(FORMATTER_NANOS) + ) + .setRefreshPolicy(IMMEDIATE) + .get(); + prepareIndex(indexNameNanoPrecision).setId("12") + .setSource( + "body", + "foo", + "@timestamp", + NOW.minusMinutes(4).withNano(randomIntBetween(0, 1_000_000_000)).format(FORMATTER_NANOS) + ) + .setRefreshPolicy(IMMEDIATE) + .get(); + prepareIndex(indexNameNanoPrecision).setId("13") + .setSource( + "body", + "foo", + "@timestamp", + NOW.minusMinutes(5).withNano(randomIntBetween(0, 1_000_000_000)).format(FORMATTER_NANOS) + ) + .setRefreshPolicy(IMMEDIATE) + .get(); + prepareIndex(indexNameNanoPrecision).setId("14") + .setSource( + "body", + "foo", + "@timestamp", + NOW.minusMinutes(6).withNano(randomIntBetween(0, 1_000_000_000)).format(FORMATTER_NANOS) + ) + .setRefreshPolicy(IMMEDIATE) + .get(); + prepareIndex(indexNameNanoPrecision).setId("15") + .setSource( + "body", + "foo", + "@timestamp", + NOW.minusMinutes(75).withNano(randomIntBetween(0, 1_000_000_000)).format(FORMATTER_NANOS) + ) + .setRefreshPolicy(IMMEDIATE) + .get(); } @After @@ -178,7 +270,7 @@ public void testOthersDottedIndexName() { SearchResponse searchResponse = client().prepareSearch("_all").setQuery(simpleQueryStringQuery("foo")).get(); try { assertNoFailures(searchResponse); - assertSearchHits(searchResponse, "1", "2"); + assertSearchHits(searchResponse, "1", "2", "1", "2", "10", "11", "12", "13", "14", "15"); } finally { searchResponse.decRef(); } @@ -227,7 +319,7 @@ public void testSimpleQuery() { } public void testSimpleQueryAgainstWildcardExpression() { - SearchResponse searchResponse = client().prepareSearch("*").setQuery(simpleQueryStringQuery("foo")).get(); + SearchResponse searchResponse = client().prepareSearch("test*").setQuery(simpleQueryStringQuery("foo")).get(); try { assertNoFailures(searchResponse); assertSearchHits(searchResponse, "1", "2"); @@ -291,8 +383,16 @@ public void testCompoundRetriever() { List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); // compound retriever does its own search as an async action, whose took time is recorded separately assertEquals(2, measurements.size()); - assertThat(measurements.getFirst().getLong(), Matchers.lessThan(searchResponse.getTook().millis())); + assertThat(measurements.getFirst().getLong(), Matchers.lessThanOrEqualTo(searchResponse.getTook().millis())); assertEquals(searchResponse.getTook().millis(), measurements.getLast().getLong()); + for (Measurement measurement : measurements) { + Map attributes = measurement.attributes(); + assertEquals(4, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals("pit", attributes.get("pit_scroll")); + } } public void testMultiSearch() { @@ -382,7 +482,13 @@ public void testTimeRangeFilterNoResults() { assertEquals(1, measurements.size()); Measurement measurement = measurements.getFirst(); assertEquals(searchResponse.getTook().millis(), measurement.getLong()); - assertTimeRangeAttributes(measurement.attributes()); + Map attributes = measurement.attributes(); + assertEquals(4, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals("@timestamp", attributes.get("time_range_filter_field")); + // there were no results, and no shards queried, hence no range filter extracted from the query either } /** @@ -404,6 +510,8 @@ public void testTimeRangeFilterAllResults() { assertEquals(1, measurements.size()); Measurement measurement = measurements.getFirst(); assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + // in this case the range query gets rewritten to a range query with open bounds on the shards. Here we test that query rewrite + // is able to grab the parsed range filter and propagate it all the way to the search response assertTimeRangeAttributes(measurement.attributes()); } @@ -427,11 +535,214 @@ public void testTimeRangeFilterOneResult() { } private static void assertTimeRangeAttributes(Map attributes) { - assertEquals(4, attributes.size()); + assertEquals(5, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals("@timestamp", attributes.get("time_range_filter_field")); + assertEquals("older_than_14_days", attributes.get("time_range_filter_from")); + } + + public void testTimeRangeFilterAllResultsFilterOnEventIngested() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("event.ingested").from("2024-10-01")); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(indexName).setPreFilterShardSize(1).setQuery(boolQueryBuilder).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1", "2"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + Map attributes = measurement.attributes(); + assertEquals(5, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals("event.ingested", attributes.get("time_range_filter_field")); + assertEquals("older_than_14_days", attributes.get("time_range_filter_from")); + } + + public void testTimeRangeFilterAllResultsFilterOnEventIngestedAndTimestamp() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("event.ingested").from("2024-10-01")); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from("2024-10-01")); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(indexName).setPreFilterShardSize(1).setQuery(boolQueryBuilder).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1", "2"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + Map attributes = measurement.attributes(); + assertEquals(5, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals("@timestamp_AND_event.ingested", attributes.get("time_range_filter_field")); + assertEquals("older_than_14_days", attributes.get("time_range_filter_from")); + } + + public void testTimeRangeFilterOneResultQueryAndFetchRecentTimestamps() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from(FORMATTER_MILLIS.format(NOW.minusMinutes(10)))); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(singleShardIndexName) + .setQuery(boolQueryBuilder) + .addSort(new FieldSortBuilder("@timestamp")) + .get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + Map attributes = measurement.attributes(); + assertEquals(5, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("@timestamp", attributes.get("sort")); + assertEquals("@timestamp", attributes.get("time_range_filter_field")); + assertEquals("15_minutes", attributes.get("time_range_filter_from")); + } + + public void testMultipleTimeRangeFiltersQueryAndFetchRecentTimestamps() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + // we take the lowest of the two bounds + boolQueryBuilder.must(new RangeQueryBuilder("@timestamp").from(FORMATTER_MILLIS.format(NOW.minusMinutes(20)))); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from(FORMATTER_MILLIS.format(NOW.minusMinutes(10)))); + // should and must_not get ignored + boolQueryBuilder.should(new RangeQueryBuilder("@timestamp").from(FORMATTER_MILLIS.format(NOW.minusMinutes(2)))); + boolQueryBuilder.mustNot(new RangeQueryBuilder("@timestamp").from(FORMATTER_MILLIS.format(NOW.minusMinutes(1)))); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(singleShardIndexName) + .setQuery(boolQueryBuilder) + .addSort(new FieldSortBuilder("@timestamp")) + .get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + Map attributes = measurement.attributes(); + assertEquals(5, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("@timestamp", attributes.get("sort")); + assertEquals("@timestamp", attributes.get("time_range_filter_field")); + assertEquals("1_hour", attributes.get("time_range_filter_from")); + } + + public void testTimeRangeFilterAllResultsShouldClause() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.should(new RangeQueryBuilder("@timestamp").from("2024-10-01")); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(indexName).setQuery(boolQueryBuilder).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1", "2"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertSimpleQueryAttributes(measurement.attributes()); + } + + public void testTimeRangeFilterOneResultMustNotClause() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.mustNot(new RangeQueryBuilder("@timestamp").from("2024-12-01")); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(indexName).setQuery(boolQueryBuilder).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertSimpleQueryAttributes(measurement.attributes()); + } + + public void testTimeRangeFilterAllResultsNanoPrecision() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from(FORMATTER_NANOS.format(NOW.minusMinutes(20)))); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(indexNameNanoPrecision).setQuery(boolQueryBuilder).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "10", "11", "12", "13", "14"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + Map attributes = measurement.attributes(); + assertEquals(5, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals("@timestamp", attributes.get("time_range_filter_field")); + assertEquals("1_hour", attributes.get("time_range_filter_from")); + } + + public void testTimeRangeFilterAllResultsMixedPrecision() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from(FORMATTER_NANOS.format(NOW.minusMinutes(20)))); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(singleShardIndexName, indexNameNanoPrecision) + .setQuery(boolQueryBuilder) + .get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1", "10", "11", "12", "13", "14"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + Map attributes = measurement.attributes(); + assertEquals(5, attributes.size()); assertEquals("user", attributes.get("target")); assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); - assertEquals(true, attributes.get("range_timestamp")); + assertEquals("@timestamp", attributes.get("time_range_filter_field")); + assertEquals("1_hour", attributes.get("time_range_filter_from")); } private void resetMeter() { @@ -439,6 +750,6 @@ private void resetMeter() { } private TestTelemetryPlugin getTestTelemetryPlugin() { - return getInstanceFromNode(PluginsService.class).filterPlugins(TestTelemetryPlugin.class).toList().get(0); + return getInstanceFromNode(PluginsService.class).filterPlugins(TestTelemetryPlugin.class).toList().getFirst(); } } diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java index 583a2e3fa5803..01c47f9ad2801 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java @@ -288,9 +288,9 @@ private static void assertTimeRangeAttributes(List measurements, St assertEquals(target, attributes.get("target")); assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); - assertEquals(true, attributes.get("range_timestamp")); + assertEquals("@timestamp", attributes.get("time_range_filter_field")); assertEquals(isSystem, attributes.get(SearchRequestAttributesExtractor.SYSTEM_THREAD_ATTRIBUTE_NAME)); - assertEquals("older_than_14_days", attributes.get("timestamp_range_filter")); + assertEquals("older_than_14_days", attributes.get("time_range_filter_from")); } } @@ -307,13 +307,30 @@ public void testTimeRangeFilterAllResults() { final List queryMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(QUERY_SEARCH_PHASE_METRIC); // the two docs are at most spread across two shards, other shards are empty and get filtered out assertThat(queryMeasurements.size(), Matchers.lessThanOrEqualTo(2)); - // no range info stored because we had no bounds after rewrite, basically a match_all - assertAttributes(queryMeasurements, false, false); + for (Measurement measurement : queryMeasurements) { + Map attributes = measurement.attributes(); + assertEquals(5, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals(false, attributes.get(SearchRequestAttributesExtractor.SYSTEM_THREAD_ATTRIBUTE_NAME)); + // the range query was rewritten to one without bounds: we do track the time range filter from value but we don't set + // the time range filter field because no range query is executed at the shard level. + assertEquals("older_than_14_days", attributes.get("time_range_filter_from")); + } final List fetchMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(FETCH_SEARCH_PHASE_METRIC); // in this case, each shard queried has results to be fetched assertEquals(queryMeasurements.size(), fetchMeasurements.size()); // no range info stored because we had no bounds after rewrite, basically a match_all - assertAttributes(fetchMeasurements, false, false); + for (Measurement measurement : fetchMeasurements) { + Map attributes = measurement.attributes(); + assertEquals(4, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals(false, attributes.get(SearchRequestAttributesExtractor.SYSTEM_THREAD_ATTRIBUTE_NAME)); + // no time range filter bucketing on the fetch phase, because the query was rewritten to one without bounds + } } private void resetMeter() { diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java index 9f2fb0d91a1cc..3c1b93e84690f 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java @@ -64,14 +64,23 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.action.search.SearchShardTask; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.cache.bitset.BitsetFilterCache; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperMetrics; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.search.ESToParentBlockJoinQuery; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardTestCase; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.lucene.queries.MinDocQuery; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -131,18 +140,52 @@ public void tearDown() throws Exception { } private TestSearchContext createContext(ContextIndexSearcher searcher, Query query) { - TestSearchContext context = new TestSearchContext(null, indexShard, searcher); + TestSearchContext context = new TestSearchContext(createSearchExecutionContext(), indexShard, searcher); context.setTask(new SearchShardTask(123L, "", "", "", null, Collections.emptyMap())); context.parsedQuery(new ParsedQuery(query)); return context; } + private SearchExecutionContext createSearchExecutionContext() { + IndexMetadata indexMetadata = IndexMetadata.builder("index") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .creationDate(System.currentTimeMillis()) + .build(); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + // final SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of()); + final long nowInMillis = randomNonNegativeLong(); + return new SearchExecutionContext( + 0, + 0, + indexSettings, + new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP), + (ft, fdc) -> ft.fielddataBuilder(fdc).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), + null, + MappingLookup.EMPTY, + null, + null, + parserConfig(), + writableRegistry(), + null, + null, + () -> nowInMillis, + null, + null, + () -> true, + null, + Collections.emptyMap(), + MapperMetrics.NOOP + ); + } + private void countTestCase(Query query, IndexReader reader, boolean shouldCollectSearch, boolean shouldCollectCount) throws Exception { ContextIndexSearcher searcher = shouldCollectSearch ? newContextSearcher(reader) : noCollectionContextSearcher(reader); try (TestSearchContext context = createContext(searcher, query)) { context.setSize(0); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); ContextIndexSearcher countSearcher = shouldCollectCount ? newContextSearcher(reader) : noCollectionContextSearcher(reader); assertEquals(countSearcher.count(query), context.queryResult().topDocs().topDocs.totalHits.value()); @@ -233,14 +276,14 @@ public void testPostFilterDisablesHitCountShortcut() throws Exception { int numDocs = indexDocs(); try (TestSearchContext context = createContext(noCollectionContextSearcher(reader), new MatchAllDocsQuery())) { context.setSize(0); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value()); assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation()); } try (TestSearchContext context = createContext(earlyTerminationContextSearcher(reader, 10), new MatchAllDocsQuery())) { // shortcutTotalHitCount makes us not track total hits as part of the top docs collection, hence size is the threshold context.setSize(10); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value()); assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation()); } @@ -257,7 +300,7 @@ public void testPostFilterDisablesHitCountShortcut() throws Exception { // shortcutTotalHitCount is disabled for filter collectors, hence we collect until track_total_hits context.setSize(10); context.parsedPostFilter(new ParsedQuery(new MatchNoDocsQuery())); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertEquals(0, context.queryResult().topDocs().topDocs.totalHits.value()); assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation()); } @@ -269,7 +312,7 @@ public void testTerminateAfterWithFilter() throws Exception { context.terminateAfter(1); context.setSize(10); context.parsedPostFilter(new ParsedQuery(new TermQuery(new Term("foo", "bar")))); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertEquals(1, context.queryResult().topDocs().topDocs.totalHits.value()); assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation()); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); @@ -280,14 +323,14 @@ public void testMinScoreDisablesHitCountShortcut() throws Exception { int numDocs = indexDocs(); try (TestSearchContext context = createContext(noCollectionContextSearcher(reader), new MatchAllDocsQuery())) { context.setSize(0); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value()); assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation()); } try (TestSearchContext context = createContext(earlyTerminationContextSearcher(reader, 10), new MatchAllDocsQuery())) { // shortcutTotalHitCount makes us not track total hits as part of the top docs collection, hence size is the threshold context.setSize(10); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value()); assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation()); } @@ -296,7 +339,7 @@ public void testMinScoreDisablesHitCountShortcut() throws Exception { // the inner TotalHitCountCollector can shortcut context.setSize(0); context.minimumScore(100); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertEquals(0, context.queryResult().topDocs().topDocs.totalHits.value()); assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation()); } @@ -313,7 +356,7 @@ public void testMinScoreDisablesHitCountShortcut() throws Exception { public void testQueryCapturesThreadPoolStats() throws Exception { indexDocs(); try (TestSearchContext context = createContext(newContextSearcher(reader), new MatchAllDocsQuery())) { - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); QuerySearchResult results = context.queryResult(); assertThat(results.serviceTimeEWMA(), greaterThanOrEqualTo(0L)); assertThat(results.nodeQueueSize(), greaterThanOrEqualTo(0)); @@ -336,7 +379,7 @@ public void testInOrderScrollOptimization() throws Exception { int size = randomIntBetween(2, 5); context.setSize(size); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs)); assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation()); assertNull(context.queryResult().terminatedEarly()); @@ -344,7 +387,7 @@ public void testInOrderScrollOptimization() throws Exception { assertThat(context.queryResult().getTotalHits().value(), equalTo((long) numDocs)); context.setSearcher(earlyTerminationContextSearcher(reader, size)); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs)); assertThat(context.queryResult().getTotalHits().value(), equalTo((long) numDocs)); assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation()); @@ -363,7 +406,7 @@ public void testTerminateAfterSize0HitCountShortcut() throws Exception { try (TestSearchContext context = createContext(noCollectionContextSearcher(reader), new MatchAllDocsQuery())) { context.terminateAfter(1); context.setSize(0); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertFalse(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs)); assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO)); @@ -476,7 +519,7 @@ public void testTerminateAfterWithHitsHitCountShortcut() throws Exception { context.terminateAfter(1); // default track_total_hits, size 1: terminate_after kicks in first context.setSize(1); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs)); assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO)); @@ -534,7 +577,7 @@ public void testTerminateAfterWithHitsNoHitCountShortcut() throws Exception { try (TestSearchContext context = createContext(earlyTerminationContextSearcher(reader, 1), query)) { context.terminateAfter(1); context.setSize(1); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo(1L)); assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO)); @@ -600,7 +643,7 @@ public void testIndexSortingEarlyTermination() throws Exception { try (TestSearchContext context = createContext(newContextSearcher(reader), new MatchAllDocsQuery())) { context.setSize(1); context.sort(new SortAndFormats(sort, new DocValueFormat[] { DocValueFormat.RAW })); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs)); assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); @@ -612,7 +655,7 @@ public void testIndexSortingEarlyTermination() throws Exception { context.setSize(1); context.sort(new SortAndFormats(sort, new DocValueFormat[] { DocValueFormat.RAW })); context.parsedPostFilter(new ParsedQuery(new MinDocQuery(1))); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertNull(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo(numDocs - 1L)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); @@ -636,7 +679,7 @@ public void testIndexSortingEarlyTermination() throws Exception { context.sort(new SortAndFormats(sort, new DocValueFormat[] { DocValueFormat.RAW })); context.setSearcher(earlyTerminationContextSearcher(reader, 1)); context.trackTotalHitsUpTo(SearchContext.TRACK_TOTAL_HITS_DISABLED); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertNull(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs[0], instanceOf(FieldDoc.class)); @@ -646,7 +689,7 @@ public void testIndexSortingEarlyTermination() throws Exception { try (TestSearchContext context = createContext(newContextSearcher(reader), new MatchAllDocsQuery())) { context.setSize(1); context.sort(new SortAndFormats(sort, new DocValueFormat[] { DocValueFormat.RAW })); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertNull(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs[0], instanceOf(FieldDoc.class)); @@ -687,7 +730,7 @@ public void testIndexSortScrollOptimization() throws Exception { context.setSize(10); context.sort(searchSortAndFormat); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs)); assertNull(context.queryResult().terminatedEarly()); assertThat(context.terminateAfter(), equalTo(0)); @@ -695,7 +738,7 @@ public void testIndexSortScrollOptimization() throws Exception { int sizeMinus1 = context.queryResult().topDocs().topDocs.scoreDocs.length - 1; FieldDoc lastDoc = (FieldDoc) context.queryResult().topDocs().topDocs.scoreDocs[sizeMinus1]; context.setSearcher(earlyTerminationContextSearcher(reader, 10)); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertNull(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs)); assertThat(context.terminateAfter(), equalTo(0)); @@ -823,7 +866,7 @@ public void testNumericSortOptimization() throws Exception { searchContext.sort(formatsLong); searchContext.trackTotalHitsUpTo(10); searchContext.setSize(10); - QueryPhase.addCollectorsAndSearch(searchContext); + QueryPhase.addCollectorsAndSearch(searchContext, null); assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints()); assertSortResults(searchContext.queryResult().topDocs().topDocs, numDocs, false); } @@ -837,7 +880,7 @@ public void testNumericSortOptimization() throws Exception { searchContext.sort(formatsLong); searchContext.trackTotalHitsUpTo(10); searchContext.setSize(10); - QueryPhase.addCollectorsAndSearch(searchContext); + QueryPhase.addCollectorsAndSearch(searchContext, null); assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints()); final TopDocs topDocs = searchContext.queryResult().topDocs().topDocs; long firstResult = (long) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]; @@ -850,7 +893,7 @@ public void testNumericSortOptimization() throws Exception { searchContext.sort(formatsLongDate); searchContext.trackTotalHitsUpTo(10); searchContext.setSize(10); - QueryPhase.addCollectorsAndSearch(searchContext); + QueryPhase.addCollectorsAndSearch(searchContext, null); assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints()); assertSortResults(searchContext.queryResult().topDocs().topDocs, numDocs, true); } @@ -860,7 +903,7 @@ public void testNumericSortOptimization() throws Exception { searchContext.sort(formatsDate); searchContext.trackTotalHitsUpTo(10); searchContext.setSize(10); - QueryPhase.addCollectorsAndSearch(searchContext); + QueryPhase.addCollectorsAndSearch(searchContext, null); assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints()); assertSortResults(searchContext.queryResult().topDocs().topDocs, numDocs, false); } @@ -870,7 +913,7 @@ public void testNumericSortOptimization() throws Exception { searchContext.sort(formatsDateLong); searchContext.trackTotalHitsUpTo(10); searchContext.setSize(10); - QueryPhase.addCollectorsAndSearch(searchContext); + QueryPhase.addCollectorsAndSearch(searchContext, null); assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints()); assertSortResults(searchContext.queryResult().topDocs().topDocs, numDocs, true); } @@ -881,7 +924,7 @@ public void testNumericSortOptimization() throws Exception { searchContext.trackTotalHitsUpTo(10); searchContext.from(5); searchContext.setSize(0); - QueryPhase.addCollectorsAndSearch(searchContext); + QueryPhase.addCollectorsAndSearch(searchContext, null); assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints()); assertThat(searchContext.queryResult().topDocs().topDocs.scoreDocs, arrayWithSize(0)); assertThat(searchContext.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs - 1)); @@ -892,7 +935,7 @@ public void testNumericSortOptimization() throws Exception { try (TestSearchContext searchContext = createContext(newContextSearcher(reader), q)) { searchContext.sort(formatsLong); searchContext.setSize(0); - QueryPhase.addCollectorsAndSearch(searchContext); + QueryPhase.addCollectorsAndSearch(searchContext, null); } } @@ -992,7 +1035,7 @@ public void testMinScore() throws Exception { context.setSize(1); context.trackTotalHitsUpTo(5); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); TotalHits totalHits = context.queryResult().topDocs().topDocs.totalHits; assertThat(totalHits.value(), greaterThanOrEqualTo(5L)); assertThat(totalHits.relation(), is(Relation.GREATER_THAN_OR_EQUAL_TO)); @@ -1047,7 +1090,7 @@ public T search(Query query, CollectorManager col } }; - try (SearchContext context = new TestSearchContext(null, indexShard, searcher) { + try (SearchContext context = new TestSearchContext(createSearchExecutionContext(), indexShard, searcher) { @Override public Query buildFilteredQuery(Query query) { return query; diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java index e19a85f4f2a0b..e3eee4dea92f0 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java @@ -43,13 +43,22 @@ import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchShardTask; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.cache.bitset.BitsetFilterCache; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.mapper.MapperMetrics; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardTestCase; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.ContextIndexSearcher; @@ -375,7 +384,7 @@ public long cost() { } private TestSearchContext createSearchContextWithTimeout(TimeoutQuery query, int size) throws IOException { - TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader)) { + TestSearchContext context = new TestSearchContext(createSearchExecutionContext(), indexShard, newContextSearcher(reader)) { @Override public long getRelativeTimeInMillis() { // this controls whether a timeout is raised or not. We abstract time away by pretending that the clock stops @@ -391,13 +400,47 @@ public long getRelativeTimeInMillis() { } private TestSearchContext createSearchContext(Query query, int size) throws IOException { - TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader)); + TestSearchContext context = new TestSearchContext(createSearchExecutionContext(), indexShard, newContextSearcher(reader)); context.setTask(new SearchShardTask(123L, "", "", "", null, Collections.emptyMap())); context.parsedQuery(new ParsedQuery(query)); context.setSize(size); return context; } + private SearchExecutionContext createSearchExecutionContext() { + IndexMetadata indexMetadata = IndexMetadata.builder("index") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .creationDate(System.currentTimeMillis()) + .build(); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + // final SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of()); + final long nowInMillis = randomNonNegativeLong(); + return new SearchExecutionContext( + 0, + 0, + indexSettings, + new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP), + (ft, fdc) -> ft.fielddataBuilder(fdc).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), + null, + MappingLookup.EMPTY, + null, + null, + parserConfig(), + writableRegistry(), + null, + null, + () -> nowInMillis, + null, + null, + () -> true, + null, + Collections.emptyMap(), + MapperMetrics.NOOP + ); + } + public void testSuggestOnlyWithTimeout() throws Exception { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().suggest(new SuggestBuilder()); try (SearchContext context = createSearchContextWithSuggestTimeout(searchSourceBuilder)) { @@ -428,7 +471,7 @@ private TestSearchContext createSearchContextWithSuggestTimeout(SearchSourceBuil ContextIndexSearcher contextIndexSearcher = newContextSearcher(reader); SuggestionSearchContext suggestionSearchContext = new SuggestionSearchContext(); suggestionSearchContext.addSuggestion("suggestion", new TestSuggestionContext(new TestSuggester(contextIndexSearcher), null)); - TestSearchContext context = new TestSearchContext(null, indexShard, contextIndexSearcher) { + TestSearchContext context = new TestSearchContext(createSearchExecutionContext(), indexShard, contextIndexSearcher) { @Override public SuggestionSearchContext suggest() { return suggestionSearchContext; diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java index e0d6189d15a8a..f47f877209d41 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java @@ -11,6 +11,8 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -58,8 +60,8 @@ public void setUpIndex() { ); ensureGreen(indexName); - prepareIndex(indexName).setId("1").setSource("body", "foo").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex(indexName).setId("2").setSource("body", "foo").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(indexName).setId("1").setSource("body", "foo", "@timestamp", "2024-11-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(indexName).setId("2").setSource("body", "foo", "@timestamp", "2024-12-01").setRefreshPolicy(IMMEDIATE).get(); } @After @@ -77,9 +79,10 @@ protected Collection> getPlugins() { * a sync request given its execution will always be completed directly as submit async search returns. */ public void testAsyncForegroundQuery() { - SubmitAsyncSearchRequest asyncSearchRequest = new SubmitAsyncSearchRequest( - new SearchSourceBuilder().query(simpleQueryStringQuery("foo")) - ); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from("2024-10-01")); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SubmitAsyncSearchRequest asyncSearchRequest = new SubmitAsyncSearchRequest(new SearchSourceBuilder().query(boolQueryBuilder)); asyncSearchRequest.setWaitForCompletionTimeout(TimeValue.timeValueSeconds(5)); asyncSearchRequest.setKeepOnCompletion(true); AsyncSearchResponse asyncSearchResponse = client().execute(SubmitAsyncSearchAction.INSTANCE, asyncSearchRequest).actionGet(); @@ -119,9 +122,10 @@ public void testAsyncForegroundQuery() { * without any influence from get async search polling happening around the same async search request. */ public void testAsyncBackgroundQuery() throws Exception { - SubmitAsyncSearchRequest asyncSearchRequest = new SubmitAsyncSearchRequest( - new SearchSourceBuilder().query(simpleQueryStringQuery("foo")) - ); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from("2024-10-01")); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SubmitAsyncSearchRequest asyncSearchRequest = new SubmitAsyncSearchRequest(new SearchSourceBuilder().query(boolQueryBuilder)); asyncSearchRequest.setWaitForCompletionTimeout(TimeValue.ZERO); AsyncSearchResponse asyncSearchResponse = client().execute(SubmitAsyncSearchAction.INSTANCE, asyncSearchRequest).actionGet(); String id; @@ -159,9 +163,11 @@ private TestTelemetryPlugin getTestTelemetryPlugin() { } private static void assertAttributes(Map attributes) { - assertEquals(3, attributes.size()); + assertEquals(5, attributes.size()); assertEquals("user", attributes.get("target")); assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); + assertEquals("@timestamp", attributes.get("time_range_filter_field")); + assertEquals("older_than_14_days", attributes.get("time_range_filter_from")); } }