Skip to content

Add relevant attributes to shard search latency APM metrics #134798

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Sep 23, 2025
Merged

Add relevant attributes to shard search latency APM metrics #134798

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
7682f37
Add relevant attributes to shard search latency APM metrics
javanna Sep 16, 2025
da04c59
Update docs/changelog/134798.yaml
javanna Sep 16, 2025
b1413f6
iter
javanna Sep 18, 2025
c575028
adjust tests
javanna Sep 18, 2025
a96f84f
[CI] Auto commit changes from spotless
Sep 18, 2025
5371640
Merge branch 'main' into enhancement/search_shard_latency_attributes
javanna Sep 19, 2025
0c0081a
Merge branch 'main' into enhancement/search_shard_latency_attributes
javanna Sep 19, 2025
77cc2b3
Merge branch 'main' into enhancement/search_shard_latency_attributes
javanna Sep 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/changelog/134798.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 134798
summary: Add relevant attributes to shard search latency APM metrics
area: Search
type: enhancement
issues: []
46 changes: 40 additions & 6 deletions server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.BoostingQueryBuilder;
import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
Expand All @@ -20,6 +22,7 @@
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.search.SearchService;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.internal.ShardSearchRequest;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.ScoreSortBuilder;
import org.elasticsearch.search.sort.SortBuilder;
Expand All @@ -42,17 +45,37 @@ private SearchRequestAttributesExtractor() {}

/**
* Introspects the provided search request and extracts metadata from it about some of its characteristics.
*
*/
public static Map<String, Object> extractAttributes(SearchRequest searchRequest, String[] localIndices) {
return extractAttributes(searchRequest.source(), searchRequest.scroll(), localIndices);
}

/**
* Introspects the provided shard search request and extracts metadata from it about some of its characteristics.
*/
public static Map<String, Object> extractAttributes(ShardSearchRequest shardSearchRequest) {
Map<String, Object> attributes = extractAttributes(
shardSearchRequest.source(),
shardSearchRequest.scroll(),
shardSearchRequest.shardId().getIndexName()
);
boolean isSystem = ((EsExecutors.EsThread) Thread.currentThread()).isSystem();
attributes.put(SYSTEM_THREAD_ATTRIBUTE_NAME, isSystem);
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The isSystem attribute I moved here from the ShardSearchPhaseAPMMetrics class.

return attributes;
}

private static Map<String, Object> extractAttributes(
SearchSourceBuilder searchSourceBuilder,
TimeValue scroll,
String... localIndices
) {
String target = extractIndices(localIndices);

String pitOrScroll = null;
if (searchRequest.scroll() != null) {
if (scroll != null) {
pitOrScroll = SCROLL;
}

SearchSourceBuilder searchSourceBuilder = searchRequest.source();
if (searchSourceBuilder == null) {
return buildAttributesMap(target, ScoreSortBuilder.NAME, HITS_ONLY, false, false, false, pitOrScroll);
}
Expand Down Expand Up @@ -144,7 +167,7 @@ private static final class QueryMetadataBuilder {
private static final String TARGET_USER = "user";
private static final String ERROR = "error";

static String extractIndices(String[] indices) {
static String extractIndices(String... indices) {
try {
// Note that indices are expected to be resolved, meaning wildcards are not handled on purpose
// If indices resolve to data streams, the name of the data stream is returned as opposed to its backing indices
Expand Down Expand Up @@ -213,6 +236,7 @@ static String extractPrimarySort(SortBuilder<?> primarySortBuilder) {
private static final String PIT = "pit";
private static final String SCROLL = "scroll";

public static final String SYSTEM_THREAD_ATTRIBUTE_NAME = "system_thread";
public static final Map<String, Object> SEARCH_SCROLL_ATTRIBUTES = Map.of(QUERY_TYPE_ATTRIBUTE, SCROLL);

static String extractQueryType(SearchSourceBuilder searchSourceBuilder) {
Expand Down Expand Up @@ -266,8 +290,18 @@ private static void introspectQueryBuilder(QueryBuilder queryBuilder, QueryMetad
break;
case RangeQueryBuilder range:
switch (range.fieldName()) {
case TIMESTAMP -> queryMetadataBuilder.rangeOnTimestamp = true;
case EVENT_INGESTED -> queryMetadataBuilder.rangeOnEventIngested = true;
// don't track unbounded ranges, they translate to either match_none if the field does not exist
// or match_all if the field is mapped
case TIMESTAMP -> {
if (range.to() != null || range.from() != null) {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is something I discovered as part of working on this chance: rewrite for range queries is a bit funky (I plan on fixing that as a follow-up). A range query may rewrite to one without bounds, if all data is within the range. In that case, it is equivalent to a match_all hence I don't report it as a range. I decided to rather report only "true" range queries at the shard level.

queryMetadataBuilder.rangeOnTimestamp = true;
}
}
case EVENT_INGESTED -> {
if (range.to() != null || range.from() != null) {
queryMetadataBuilder.rangeOnEventIngested = true;
}
}
}
break;
case KnnVectorQueryBuilder knn:
Expand Down
21 changes: 7 additions & 14 deletions server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,13 @@

package org.elasticsearch.index.search.stats;

import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.action.search.SearchRequestAttributesExtractor;
import org.elasticsearch.index.shard.SearchOperationListener;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.search.internal.ShardSearchRequest;
import org.elasticsearch.telemetry.metric.LongHistogram;
import org.elasticsearch.telemetry.metric.MeterRegistry;

import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;

Expand All @@ -24,14 +24,9 @@ public final class ShardSearchPhaseAPMMetrics implements SearchOperationListener
public static final String QUERY_SEARCH_PHASE_METRIC = "es.search.shards.phases.query.duration.histogram";
public static final String FETCH_SEARCH_PHASE_METRIC = "es.search.shards.phases.fetch.duration.histogram";

public static final String SYSTEM_THREAD_ATTRIBUTE_NAME = "system_thread";

private final LongHistogram queryPhaseMetric;
private final LongHistogram fetchPhaseMetric;

// Avoid allocating objects in the search path and multithreading clashes
private static final ThreadLocal<Map<String, Object>> THREAD_LOCAL_ATTRS = ThreadLocal.withInitial(() -> new HashMap<>(1));

public ShardSearchPhaseAPMMetrics(MeterRegistry meterRegistry) {
this.queryPhaseMetric = meterRegistry.registerLongHistogram(
QUERY_SEARCH_PHASE_METRIC,
Expand All @@ -47,18 +42,16 @@ public ShardSearchPhaseAPMMetrics(MeterRegistry meterRegistry) {

@Override
public void onQueryPhase(SearchContext searchContext, long tookInNanos) {
recordPhaseLatency(queryPhaseMetric, tookInNanos);
recordPhaseLatency(queryPhaseMetric, tookInNanos, searchContext.request());
}

@Override
public void onFetchPhase(SearchContext searchContext, long tookInNanos) {
recordPhaseLatency(fetchPhaseMetric, tookInNanos);
recordPhaseLatency(fetchPhaseMetric, tookInNanos, searchContext.request());
}

private static void recordPhaseLatency(LongHistogram histogramMetric, long tookInNanos) {
Map<String, Object> attrs = ShardSearchPhaseAPMMetrics.THREAD_LOCAL_ATTRS.get();
boolean isSystem = ((EsExecutors.EsThread) Thread.currentThread()).isSystem();
attrs.put(SYSTEM_THREAD_ATTRIBUTE_NAME, isSystem);
histogramMetric.record(TimeUnit.NANOSECONDS.toMillis(tookInNanos), attrs);
private static void recordPhaseLatency(LongHistogram histogramMetric, long tookInNanos, ShardSearchRequest request) {
Map<String, Object> attributes = SearchRequestAttributesExtractor.extractAttributes(request);
histogramMetric.record(TimeUnit.NANOSECONDS.toMillis(tookInNanos), attributes);
}
}
36 changes: 25 additions & 11 deletions .../src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,18 @@ public void testExtractAttributes() {
searchRequest,
searchRequest.indices()
);
assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, false, false, null);
}
{
SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchRequest.source(searchSourceBuilder);
searchSourceBuilder.sort("@timestamp");
searchSourceBuilder.query(new RangeQueryBuilder("@timestamp").from("2021-11-11"));
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
searchRequest,
searchRequest.indices()
);
assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null);
}
{
Expand All @@ -202,10 +214,10 @@ public void testExtractAttributes() {
boolQueryBuilder.must(boolQueryBuilderNew);
boolQueryBuilder = boolQueryBuilderNew;
}
boolQueryBuilder.must(new RangeQueryBuilder("@timestamp"));
boolQueryBuilder.must(new RangeQueryBuilder("@timestamp").from("2021-11-11"));
searchSourceBuilder.query(boolQueryBuilder);
if (randomBoolean()) {
boolQueryBuilder.should(new RangeQueryBuilder("event.ingested"));
boolQueryBuilder.should(new RangeQueryBuilder("event.ingested").from("2021-11-11"));
}
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
searchRequest,
Expand All @@ -229,7 +241,7 @@ public void testExtractAttributes() {
boolQueryBuilder.should(new RangeQueryBuilder("event.ingested"));
}

boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp"));
boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from("2021-11-11"));
searchSourceBuilder.query(boolQueryBuilder);
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
searchRequest,
Expand All @@ -243,8 +255,8 @@ public void testExtractAttributes() {
searchRequest.source(searchSourceBuilder);
searchSourceBuilder.sort("@timestamp");
BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
boolQueryBuilder.must(new RangeQueryBuilder("@timestamp"));
boolQueryBuilder.must(new RangeQueryBuilder("event.ingested"));
boolQueryBuilder.must(new RangeQueryBuilder("@timestamp").from("2021-11-11"));
boolQueryBuilder.must(new RangeQueryBuilder("event.ingested").from("2021-11-11"));
boolQueryBuilder.must(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10)));
searchSourceBuilder.query(boolQueryBuilder);
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
Expand All @@ -259,7 +271,7 @@ public void testExtractAttributes() {
searchRequest.source(searchSourceBuilder);
searchSourceBuilder.sort("@timestamp");
BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
boolQueryBuilder.should(new RangeQueryBuilder("@timestamp"));
boolQueryBuilder.should(new RangeQueryBuilder("@timestamp").from("2021-11-11"));
searchSourceBuilder.query(boolQueryBuilder);
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
searchRequest,
Expand All @@ -273,7 +285,7 @@ public void testExtractAttributes() {
searchRequest.source(searchSourceBuilder);
searchSourceBuilder.sort("@timestamp");
BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
boolQueryBuilder.should(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10)));
boolQueryBuilder.should(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10)).from("2021-11-11"));
searchSourceBuilder.query(boolQueryBuilder);
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
searchRequest,
Expand All @@ -286,7 +298,7 @@ public void testExtractAttributes() {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchRequest.source(searchSourceBuilder);
searchSourceBuilder.sort("@timestamp");
searchSourceBuilder.query(new ConstantScoreQueryBuilder(new RangeQueryBuilder("@timestamp")));
searchSourceBuilder.query(new ConstantScoreQueryBuilder(new RangeQueryBuilder("@timestamp").from("2021-11-11")));
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
searchRequest,
searchRequest.indices()
Expand All @@ -298,7 +310,9 @@ public void testExtractAttributes() {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchRequest.source(searchSourceBuilder);
searchSourceBuilder.sort("@timestamp");
searchSourceBuilder.query(new BoostingQueryBuilder(new RangeQueryBuilder("@timestamp"), new MatchAllQueryBuilder()));
searchSourceBuilder.query(
new BoostingQueryBuilder(new RangeQueryBuilder("@timestamp").from("2021-11-11"), new MatchAllQueryBuilder())
);
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
searchRequest,
searchRequest.indices()
Expand All @@ -320,7 +334,7 @@ public void testDepthLimit() {
newBoolQueryBuilder.must(innerBoolQueryBuilder);
newBoolQueryBuilder = innerBoolQueryBuilder;
}
newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp"));
newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp").from("2021-11-11"));
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
searchRequest,
searchRequest.indices()
Expand All @@ -339,7 +353,7 @@ public void testDepthLimit() {
newBoolQueryBuilder.must(innerBoolQueryBuilder);
newBoolQueryBuilder = innerBoolQueryBuilder;
}
newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp"));
newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp").from("2021-11-11"));
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
searchRequest,
searchRequest.indices()
Expand Down
Loading