Skip to content

Commit

Permalink
Expose size statistics for completion suggest
Browse files Browse the repository at this point in the history 
In order to determine how many RAM the completion suggest structures will eat up, this data should be exposed.

Closes #3522
  • Loading branch information
@spinscale
spinscale committed Aug 21, 2013
1 parent ac3d5d6 commit cdddbb7
Show file tree
Hide file tree
Showing 14 changed files with 416 additions and 10 deletions.
34 changes: 34 additions & 0 deletions src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
View file
Expand Up @@ -19,6 +19,7 @@

package org.elasticsearch.action.admin.indices.stats;

import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
Expand All @@ -38,6 +39,7 @@
import org.elasticsearch.index.shard.DocsStats;
import org.elasticsearch.index.store.StoreStats;
import org.elasticsearch.index.warmer.WarmerStats;
import org.elasticsearch.search.suggest.completion.CompletionStats;

import java.io.IOException;

Expand Down Expand Up @@ -84,6 +86,9 @@ public class CommonStats implements Streamable, ToXContent {
@Nullable
public PercolateStats percolate;

@Nullable
public CompletionStats completion;

public void add(CommonStats stats) {
if (docs == null) {
if (stats.getDocs() != null) {
Expand Down Expand Up @@ -191,6 +196,14 @@ public void add(CommonStats stats) {
} else {
percolate.add(stats.getPercolate());
}
if (completion == null) {
if (stats.getCompletion() != null) {
completion = new CompletionStats();
completion.add(stats.getCompletion());
}
} else {
completion.add(stats.getCompletion());
}
}

@Nullable
Expand Down Expand Up @@ -258,6 +271,11 @@ public PercolateStats getPercolate() {
return percolate;
}

@Nullable
public CompletionStats getCompletion() {
return completion;
}

public static CommonStats readCommonStats(StreamInput in) throws IOException {
CommonStats stats = new CommonStats();
stats.readFrom(in);
Expand Down Expand Up @@ -305,6 +323,11 @@ public void readFrom(StreamInput in) throws IOException {
if (in.readBoolean()) {
percolate = PercolateStats.readPercolateStats(in);
}
if (in.getVersion().onOrAfter(Version.V_0_90_4)) {
if (in.readBoolean()) {
completion = CompletionStats.readCompletionStats(in);
}
}
}

@Override
Expand Down Expand Up @@ -387,6 +410,14 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeBoolean(true);
percolate.writeTo(out);
}
if (out.getVersion().onOrAfter(Version.V_0_90_4)) {
if (completion == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
completion.writeTo(out);
}
}
}

// note, requires a wrapping object
Expand Down Expand Up @@ -431,6 +462,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (percolate != null) {
percolate.toXContent(builder, params);
}
if (completion != null) {
completion.toXContent(builder, params);
}
return builder;
}
}
21 changes: 20 additions & 1 deletion src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java
View file
Expand Up @@ -19,6 +19,7 @@

package org.elasticsearch.action.admin.indices.stats;

import org.elasticsearch.Version;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
Expand All @@ -33,6 +34,7 @@ public class CommonStatsFlags implements Streamable, Cloneable {
private String[] types = null;
private String[] groups = null;
private String[] fieldDataFields = null;
private String[] completionDataFields = null;

/**
* Sets all flags to return all stats.
Expand All @@ -42,6 +44,7 @@ public CommonStatsFlags all() {
types = null;
groups = null;
fieldDataFields = null;
completionDataFields = null;
return this;
}

Expand All @@ -53,6 +56,7 @@ public CommonStatsFlags clear() {
types = null;
groups = null;
fieldDataFields = null;
completionDataFields = null;
return this;
}

Expand Down Expand Up @@ -107,6 +111,14 @@ public String[] fieldDataFields() {
return this.fieldDataFields;
}

public CommonStatsFlags completionDataFields(String... completionDataFields) {
this.completionDataFields = completionDataFields;
return this;
}

public String[] completionDataFields() {
return this.completionDataFields;
}

public boolean isSet(Flag flag) {
return flags.contains(flag);
Expand Down Expand Up @@ -146,6 +158,9 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeStringArrayNullable(types);
out.writeStringArrayNullable(groups);
out.writeStringArrayNullable(fieldDataFields);
if (out.getVersion().onOrAfter(Version.V_0_90_4)) {
out.writeStringArrayNullable(completionDataFields);
}
}

@Override
Expand All @@ -160,6 +175,9 @@ public void readFrom(StreamInput in) throws IOException {
types = in.readStringArray();
groups = in.readStringArray();
fieldDataFields = in.readStringArray();
if (in.getVersion().onOrAfter(Version.V_0_90_4)) {
completionDataFields = in.readStringArray();
}
}

@Override
Expand Down Expand Up @@ -188,7 +206,8 @@ public static enum Flag {
FieldData("fielddata"),
Docs("docs"),
Warmer("warmer"),
Percolate("percolate");
Percolate("percolate"),
Completion("completion");

private final String restName;

Expand Down
18 changes: 18 additions & 0 deletions src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java
View file
Expand Up @@ -212,6 +212,24 @@ public String[] fieldDataFields() {
return flags.fieldDataFields();
}

public IndicesStatsRequest completion(boolean completion) {
flags.set(Flag.Completion, completion);
return this;
}

public boolean completion() {
return flags.isSet(Flag.Completion);
}

public IndicesStatsRequest completionFields(String ... completionDataFields) {
flags.completionDataFields(completionDataFields);
return this;
}

public String[] completionFields() {
return flags.completionDataFields();
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
Expand Down
10 changes: 10 additions & 0 deletions src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java
View file
Expand Up @@ -139,6 +139,16 @@ public IndicesStatsRequestBuilder setPercolate(boolean percolate) {
return this;
}

public IndicesStatsRequestBuilder setCompletion(boolean completion) {
request.completion(completion);
return this;
}

public IndicesStatsRequestBuilder setCompletionFields(String... fields) {
request.completionFields(fields);
return this;
}

@Override
protected void doExecute(ActionListener<IndicesStatsResponse> listener) {
((IndicesAdminClient) client).stats(request, listener);
Expand Down
3 changes: 3 additions & 0 deletions src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java
View file
Expand Up @@ -179,6 +179,9 @@ protected ShardStats shardOperation(IndexShardStatsRequest request) throws Elast
if (request.request.percolate()) {
stats.stats.percolate = indexShard.shardPercolateService().stats();
}
if (request.request.completion()) {
stats.stats.completion = indexShard.completionStats(request.request.completionFields());
}

return stats;
}
Expand Down
3 changes: 3 additions & 0 deletions src/main/java/org/elasticsearch/index/shard/service/IndexShard.java
View file
Expand Up @@ -50,6 +50,7 @@
import org.elasticsearch.index.store.StoreStats;
import org.elasticsearch.index.warmer.ShardIndexWarmerService;
import org.elasticsearch.index.warmer.WarmerStats;
import org.elasticsearch.search.suggest.completion.CompletionStats;

/**
*
Expand Down Expand Up @@ -96,6 +97,8 @@ public interface IndexShard extends IndexShardComponent {

FieldDataStats fieldDataStats(String... fields);

CompletionStats completionStats(String ... fields);

PercolatorQueriesRegistry percolateRegistry();

ShardPercolateService shardPercolateService();
Expand Down
22 changes: 22 additions & 0 deletions src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java
View file
Expand Up @@ -20,6 +20,7 @@
package org.elasticsearch.index.shard.service;

import com.google.common.base.Charsets;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.index.CheckIndex;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.Query;
Expand Down Expand Up @@ -75,6 +76,8 @@
import org.elasticsearch.indices.IndicesLifecycle;
import org.elasticsearch.indices.InternalIndicesLifecycle;
import org.elasticsearch.indices.recovery.RecoveryStatus;
import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat;
import org.elasticsearch.search.suggest.completion.CompletionStats;
import org.elasticsearch.threadpool.ThreadPool;

import java.io.IOException;
Expand Down Expand Up @@ -506,6 +509,25 @@ public IdCacheStats idCacheStats() {
return shardIdCache.stats();
}

@Override
public CompletionStats completionStats(String... fields) {
CompletionStats completionStats = new CompletionStats();

Engine.Searcher searcher = engine().searcher();

try {
PostingsFormat postingsFormat = this.codecService.postingsFormatService().get(Completion090PostingsFormat.CODEC_NAME).get();
if (postingsFormat instanceof Completion090PostingsFormat) {
Completion090PostingsFormat completionPostingsFormat = (Completion090PostingsFormat) postingsFormat;
completionStats.add(completionPostingsFormat.completionStats(searcher().reader(), fields));
}
} finally {
searcher.release();
}

return completionStats;
}

@Override
public void flush(Engine.Flush flush) throws ElasticSearchException {
// we allows flush while recovering, since we allow for operations to happen
Expand Down
7 changes: 7 additions & 0 deletions src/main/java/org/elasticsearch/indices/InternalIndicesService.java
View file
Expand Up @@ -76,6 +76,7 @@
import org.elasticsearch.indices.store.IndicesStore;
import org.elasticsearch.plugins.IndexPluginsModule;
import org.elasticsearch.plugins.PluginsService;
import org.elasticsearch.search.suggest.completion.CompletionStats;

import java.util.HashMap;
import java.util.Map;
Expand Down Expand Up @@ -242,6 +243,9 @@ public NodeIndicesStats stats(boolean includePrevious, CommonStatsFlags flags) {
case Percolate:
stats.percolate = new PercolateStats();
break;
case Completion:
stats.completion = new CompletionStats();
break;
default:
throw new IllegalStateException("Unknown Flag: " + flag);
}
Expand Down Expand Up @@ -291,6 +295,9 @@ public NodeIndicesStats stats(boolean includePrevious, CommonStatsFlags flags) {
case Percolate:
stats.percolate.add(indexShard.shardPercolateService().stats());
break;
case Completion:
stats.completion.add(indexShard.completionStats(flags.completionDataFields()));
break;
default:
throw new IllegalStateException("Unknown Flag: " + flag);
}
Expand Down
44 changes: 44 additions & 0 deletions src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java
View file
Expand Up @@ -89,6 +89,11 @@ public RestIndicesStatsAction(Settings settings, Client client, RestController c
controller.registerHandler(GET, "/_stats/fielddata/{fields}", new RestFieldDataStatsHandler());
controller.registerHandler(GET, "/{index}/_stats/fielddata/{fields}", new RestFieldDataStatsHandler());

controller.registerHandler(GET, "/_stats/completion", new RestCompletionStatsHandler());
controller.registerHandler(GET, "/{index}/_stats/completion", new RestCompletionStatsHandler());
controller.registerHandler(GET, "/_stats/completion/{fields}", new RestCompletionStatsHandler());
controller.registerHandler(GET, "/{index}/_stats/completion/{fields}", new RestCompletionStatsHandler());

controller.registerHandler(GET, "/_stats/percolate", new RestPercolateStatsHandler());
controller.registerHandler(GET, "/{index}/_stats/percolate", new RestPercolateStatsHandler());
}
Expand Down Expand Up @@ -585,6 +590,45 @@ public void onFailure(Throwable e) {
}
}

class RestCompletionStatsHandler implements RestHandler {

@Override
public void handleRequest(final RestRequest request, final RestChannel channel) {
IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest();
indicesStatsRequest.listenerThreaded(false);
indicesStatsRequest.clear().completion(true);
indicesStatsRequest.indices(splitIndices(request.param("index")));
indicesStatsRequest.types(splitTypes(request.param("types")));
indicesStatsRequest.completionFields(request.paramAsStringArray("fields", null));

client.admin().indices().stats(indicesStatsRequest, new ActionListener<IndicesStatsResponse>() {
@Override
public void onResponse(IndicesStatsResponse response) {
try {
XContentBuilder builder = RestXContentBuilder.restContentBuilder(request);
builder.startObject();
builder.field("ok", true);
buildBroadcastShardsHeader(builder, response);
response.toXContent(builder, request);
builder.endObject();
channel.sendResponse(new XContentRestResponse(request, OK, builder));
} catch (Throwable e) {
onFailure(e);
}
}

@Override
public void onFailure(Throwable e) {
try {
channel.sendResponse(new XContentThrowableRestResponse(request, e));
} catch (IOException e1) {
logger.error("Failed to send failure response", e1);
}
}
});
}
}

class RestRefreshStatsHandler implements RestHandler {

@Override
Expand Down

0 comments on commit cdddbb7

Please sign in to comment.