Skip to content

Commit

Permalink
add to stats API
Browse files Browse the repository at this point in the history
  • Loading branch information
@ywelsch
ywelsch committed Jun 17, 2021
1 parent 99a3c6b commit ba22ce0
Show file tree
Hide file tree
Showing 11 changed files with 536 additions and 151 deletions.
84 changes: 42 additions & 42 deletions server/src/internalClusterTest/java/org/elasticsearch/search/stats/FieldUsageStatsIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,9 @@

import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.search.stats.FieldUsageStats;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.internal.FieldUsageTrackingDirectoryReader.UsageContext;
import org.elasticsearch.test.ESSingleNodeTestCase;

import java.util.Set;
Expand All @@ -35,30 +31,24 @@ protected Settings nodeSettings() {
}

public void testFieldUsageStats() {
int numShards = 2;
assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder()
.put(SETTING_NUMBER_OF_SHARDS, 2)
.put(SETTING_NUMBER_OF_SHARDS, numShards)
.put(SETTING_NUMBER_OF_REPLICAS, 0)));
IndexShard indexShard = null;
for (IndexService indexService : getInstanceFromNode(IndicesService.class)) {
if (indexService.index().getName().equals("test")) {
indexShard = indexService.getShard(0);
break;
}
}

assertNotNull(indexShard);
for (int i = 1; i < 10; i++) {
client().prepareIndex("test").setId(Integer.toString(i)).setSource(
"field", "value", "field2", "value2", "date_field", "2015/09/0" + i).get();
}
client().admin().indices().prepareRefresh("test").get();

final FieldUsageStats stats = indexShard.fieldUsageStats();
FieldUsageStats stats = client().admin().indices().prepareStats("test").clear().setFieldUsage(true).get()
.getIndex("test").getTotal().getFieldUsageStats();

assertFalse(stats.getPerFieldStats().containsKey("field"));
assertFalse(stats.getPerFieldStats().containsKey("field.keyword"));
assertFalse(stats.getPerFieldStats().containsKey("field2"));
assertFalse(stats.getPerFieldStats().containsKey("date_field"));
assertFalse(stats.hasField("field"));
assertFalse(stats.hasField("field.keyword"));
assertFalse(stats.hasField("field2"));
assertFalse(stats.hasField("date_field"));

SearchResponse searchResponse = client().prepareSearch()
.setQuery(QueryBuilders.termQuery("field", "value"))
Expand All @@ -69,57 +59,67 @@ public void testFieldUsageStats() {
assertHitCount(searchResponse, 9);
assertAllSuccessful(searchResponse);

logger.info("Stats after first query: {}", stats.getPerFieldStats());
stats = client().admin().indices().prepareStats("test").clear().setFieldUsage(true).get()
.getIndex("test").getTotal().getFieldUsageStats();
logger.info("Stats after first query: {}", stats);

assertTrue(stats.getPerFieldStats().containsKey("_id"));
assertEquals(Set.of(UsageContext.STORED_FIELDS), stats.getPerFieldStats().get("_id").keySet());
assertTrue(stats.getPerFieldStats().containsKey("_source"));
assertEquals(Set.of(UsageContext.STORED_FIELDS), stats.getPerFieldStats().get("_source").keySet());
assertTrue(stats.hasField("_id"));
assertEquals(Set.of(FieldUsageStats.UsageContext.STORED_FIELDS), stats.get("_id").keySet());
assertTrue(stats.hasField("_source"));
assertEquals(Set.of(FieldUsageStats.UsageContext.STORED_FIELDS), stats.get("_source").keySet());

assertTrue(stats.getPerFieldStats().containsKey("field"));
assertTrue(stats.hasField("field"));
// we sort by _score
assertEquals(Set.of(UsageContext.TERMS, UsageContext.FREQS, UsageContext.NORMS), stats.getPerFieldStats().get("field").keySet());
assertEquals(1L, stats.getPerFieldStats().get("field").get(UsageContext.TERMS));
assertEquals(Set.of(FieldUsageStats.UsageContext.TERMS, FieldUsageStats.UsageContext.FREQS, FieldUsageStats.UsageContext.NORMS),
stats.get("field").keySet());
assertEquals(1L * numShards, stats.get("field").getTerms());

assertTrue(stats.getPerFieldStats().containsKey("field2"));
assertTrue(stats.hasField("field2"));
// positions because of span query
assertEquals(Set.of(UsageContext.TERMS, UsageContext.FREQS, UsageContext.POSITIONS),
stats.getPerFieldStats().get("field2").keySet());
assertEquals(1L, stats.getPerFieldStats().get("field2").get(UsageContext.TERMS));
assertEquals(Set.of(FieldUsageStats.UsageContext.TERMS, FieldUsageStats.UsageContext.FREQS, FieldUsageStats.UsageContext.POSITIONS),
stats.get("field2").keySet());
assertEquals(1L * numShards, stats.get("field2").getTerms());

assertTrue(stats.getPerFieldStats().containsKey("field.keyword"));
assertTrue(stats.hasField("field.keyword"));
// terms agg does not use search as we've set search.aggs.rewrite_to_filter_by_filter to false
assertEquals(Set.of(UsageContext.DOC_VALUES), stats.getPerFieldStats().get("field.keyword").keySet());
assertEquals(1L, stats.getPerFieldStats().get("field.keyword").get(UsageContext.DOC_VALUES));
assertEquals(Set.of(FieldUsageStats.UsageContext.DOC_VALUES), stats.get("field.keyword").keySet());
assertEquals(1L * numShards, stats.get("field.keyword").getDocValues());

client().prepareSearch()
.setQuery(QueryBuilders.termQuery("field", "value"))
.addAggregation(AggregationBuilders.terms("agg1").field("field.keyword"))
.setSize(100)
.get();

logger.info("Stats after second query: {}", stats.getPerFieldStats());
stats = client().admin().indices().prepareStats("test").clear().setFieldUsage(true).get()
.getIndex("test").getTotal().getFieldUsageStats();
logger.info("Stats after second query: {}", stats);

assertEquals(2L, stats.getPerFieldStats().get("field").get(UsageContext.TERMS));
assertEquals(1L, stats.getPerFieldStats().get("field2").get(UsageContext.TERMS));
assertEquals(2L, stats.getPerFieldStats().get("field.keyword").get(UsageContext.DOC_VALUES));
assertEquals(2L * numShards, stats.get("field").getTerms());
assertEquals(1L * numShards, stats.get("field2").getTerms());
assertEquals(2L * numShards, stats.get("field.keyword").getDocValues());

assertFalse(stats.getPerFieldStats().containsKey("date_field"));
assertFalse(stats.hasField("date_field"));

// show that we also track stats in can_match
assertEquals(4, client().admin().indices().prepareStats("test").clear().setSearch(true).get()
assertEquals(2L * numShards, client().admin().indices().prepareStats("test").clear().setSearch(true).get()
.getIndex("test").getTotal().getSearch().getTotal().getQueryCount());
client().prepareSearch()
.setPreFilterShardSize(1)
.setQuery(QueryBuilders.rangeQuery("date_field").from("2016/01/01"))
.setSize(100)
.get();
assertTrue(stats.getPerFieldStats().containsKey("date_field"));
assertEquals(Set.of(UsageContext.POINTS), stats.getPerFieldStats().get("date_field").keySet());

stats = client().admin().indices().prepareStats("test").clear().setFieldUsage(true).get()
.getIndex("test").getTotal().getFieldUsageStats();
logger.info("Stats after third query: {}", stats);

assertTrue(stats.hasField("date_field"));
assertEquals(Set.of(FieldUsageStats.UsageContext.POINTS), stats.get("date_field").keySet());
// can_match does not enter search stats
// there is a special case though where we have no hit but we need to get at least one search response in order
// to produce a valid search result with all the aggs etc., so we hit one of the two shards
assertEquals(5, client().admin().indices().prepareStats("test").clear().setSearch(true).get()
assertEquals((2 * numShards) + 1, client().admin().indices().prepareStats("test").clear().setSearch(true).get()
.getIndex("test").getTotal().getSearch().getTotal().getQueryCount());
}
}
35 changes: 32 additions & 3 deletions server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,15 @@

import org.apache.lucene.store.AlreadyClosedException;
import org.elasticsearch.Version;
import org.elasticsearch.index.bulk.stats.BulkStats;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.index.bulk.stats.BulkStats;
import org.elasticsearch.index.cache.query.QueryCacheStats;
import org.elasticsearch.index.cache.request.RequestCacheStats;
import org.elasticsearch.index.engine.SegmentsStats;
Expand All @@ -28,6 +28,7 @@
import org.elasticsearch.index.merge.MergeStats;
import org.elasticsearch.index.recovery.RecoveryStats;
import org.elasticsearch.index.refresh.RefreshStats;
import org.elasticsearch.index.search.stats.FieldUsageStats;
import org.elasticsearch.index.search.stats.SearchStats;
import org.elasticsearch.index.shard.DocsStats;
import org.elasticsearch.index.shard.IndexShard;
Expand Down Expand Up @@ -96,6 +97,9 @@ public class CommonStats implements Writeable, ToXContentFragment {
@Nullable
public BulkStats bulk;

@Nullable
public FieldUsageStats fieldUsageStats;

public CommonStats() {
this(CommonStatsFlags.NONE);
}
Expand Down Expand Up @@ -156,6 +160,9 @@ public CommonStats(CommonStatsFlags flags) {
case Bulk:
bulk = new BulkStats();
break;
case FieldUsage:
fieldUsageStats = new FieldUsageStats();
break;
default:
throw new IllegalStateException("Unknown Flag: " + flag);
}
Expand Down Expand Up @@ -218,6 +225,9 @@ public CommonStats(IndicesQueryCache indicesQueryCache, IndexShard indexShard, C
case Bulk:
bulk = indexShard.bulkStats();
break;
case FieldUsage:
fieldUsageStats = indexShard.fieldUsageTracker().stats();
break;
default:
throw new IllegalStateException("Unknown Flag: " + flag);
}
Expand Down Expand Up @@ -247,6 +257,9 @@ public CommonStats(StreamInput in) throws IOException {
if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
bulk = in.readOptionalWriteable(BulkStats::new);
}
if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
fieldUsageStats = in.readOptionalWriteable(FieldUsageStats::new);
}
}

@Override
Expand All @@ -270,6 +283,9 @@ public void writeTo(StreamOutput out) throws IOException {
if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
out.writeOptionalWriteable(bulk);
}
if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
out.writeOptionalWriteable(fieldUsageStats);
}
}

public void add(CommonStats stats) {
Expand Down Expand Up @@ -410,6 +426,14 @@ public void add(CommonStats stats) {
} else {
bulk.add(stats.getBulk());
}
if (fieldUsageStats == null) {
if (stats.fieldUsageStats != null) {
fieldUsageStats = new FieldUsageStats();
fieldUsageStats.add(stats.getFieldUsageStats());
}
} else {
fieldUsageStats.add(stats.getFieldUsageStats());
}
}

@Nullable
Expand Down Expand Up @@ -497,6 +521,11 @@ public BulkStats getBulk() {
return bulk;
}

@Nullable
public FieldUsageStats getFieldUsageStats() {
return fieldUsageStats;
}

/**
* Utility method which computes total memory by adding
* FieldData, PercolatorCache, Segments (memory, index writer, version map)
Expand All @@ -523,7 +552,7 @@ public ByteSizeValue getTotalMemory() {
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
final Stream<ToXContent> stream = Arrays.stream(new ToXContent[] {
docs, store, indexing, get, search, merge, refresh, flush, warmer, queryCache,
fieldData, completion, segments, translog, requestCache, recoveryStats, bulk})
fieldData, completion, segments, translog, requestCache, recoveryStats, bulk, fieldUsageStats})
.filter(Objects::nonNull);
for (ToXContent toXContent : ((Iterable<ToXContent>)stream::iterator)) {
toXContent.toXContent(builder, params);
Expand Down
3 changes: 2 additions & 1 deletion server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,8 @@ public enum Flag {
// 14 was previously used for Suggest
RequestCache("request_cache", 15),
Recovery("recovery", 16),
Bulk("bulk", 17);
Bulk("bulk", 17),
FieldUsage("field_usage", 18);

private final String restName;
private final int index;
Expand Down
9 changes: 9 additions & 0 deletions server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,15 @@ public boolean bulk() {
return flags.isSet(Flag.Bulk);
}

public IndicesStatsRequest fieldUsage(boolean fieldUsage) {
flags.set(Flag.FieldUsage, fieldUsage);
return this;
}

public boolean fieldUsage() {
return flags.isSet(Flag.FieldUsage);
}

public boolean includeSegmentFileSizes() {
return flags.includeSegmentFileSizes();
}
Expand Down
5 changes: 5 additions & 0 deletions ...rc/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,11 @@ public IndicesStatsRequestBuilder setBulk(boolean bulk) {
return this;
}

public IndicesStatsRequestBuilder setFieldUsage(boolean fieldUsage) {
request.fieldUsage(fieldUsage);
return this;
}

public IndicesStatsRequestBuilder setIncludeSegmentFileSizes(boolean includeSegmentFileSizes) {
request.includeSegmentFileSizes(includeSegmentFileSizes);
return this;
Expand Down
Loading

0 comments on commit ba22ce0

Please sign in to comment.