Skip to content

Commit

Permalink
Simplify nodes stats API
Browse files Browse the repository at this point in the history 
First, this breaks backwards compatibility!

* Removed /_cluster/nodes/stats endpoint
* Excpect the stats types not as parameters, but as part of the URL
* Returning all indices stats by default, returning all nodes stats by default
* Supporting groups & types in nodes stats now as well
* Updated documentation & tests accordingly
* Allow level parameter for "shards" and "indices" (cluster does not make sense here)

Closes #4057
  • Loading branch information
@spinscale
spinscale committed Jan 6, 2014
1 parent 33878be commit bb27516
Show file tree
Hide file tree
Showing 5 changed files with 213 additions and 231 deletions.
21 changes: 7 additions & 14 deletions docs/reference/cluster/nodes-stats.asciidoc
View file
Expand Up @@ -9,10 +9,6 @@ the cluster nodes statistics.

[source,js]
--------------------------------------------------
curl -XGET 'http://localhost:9200/_cluster/nodes/stats'
curl -XGET 'http://localhost:9200/_cluster/nodes/nodeId1,nodeId2/stats'
# simplified
curl -XGET 'http://localhost:9200/_nodes/stats'
curl -XGET 'http://localhost:9200/_nodes/nodeId1,nodeId2/stats'
--------------------------------------------------
Expand All @@ -22,9 +18,9 @@ second command selectively retrieves nodes stats of only `nodeId1` and
`nodeId2`. All the nodes selective options are explained
<<cluster-nodes,here>>.

By default, `indices` stats are returned. With options for `indices`,
`os`, `process`, `jvm`, `network`, `transport`, `http`, `fs`, `breaker`, and
`thread_pool`. For example:
By default, all stats are returned. You can limit this by combining any
of `indices`, `os`, `process`, `jvm`, `network`, `transport`, `http`,
`fs`, `breaker` and `thread_pool`. For example:

[horizontal]
`indices`::
Expand Down Expand Up @@ -70,13 +66,10 @@ By default, `indices` stats are returned. With options for `indices`,
[source,js]
--------------------------------------------------
# return indices and os
curl -XGET 'http://localhost:9200/_nodes/stats?os=true'
curl -XGET 'http://localhost:9200/_nodes/stats/os'
# return just os and process
curl -XGET 'http://localhost:9200/_nodes/stats?clear=true&os=true&process=true'
curl -XGET 'http://localhost:9200/_nodes/stats/os,process'
# specific type endpoint
curl -XGET 'http://localhost:9200/_nodes/process/stats'
curl -XGET 'http://localhost:9200/_nodes/10.0.0.1/process/stats'
# or, if you like the other way
curl -XGET 'http://localhost:9200/_nodes/stats/process'
curl -XGET 'http://localhost:9200/_nodes/10.0.0.1/stats/process'
--------------------------------------------------
Expand All @@ -93,12 +86,12 @@ level or on index level.
[source,js]
--------------------------------------------------
# Node Stats
curl localhost:9200/_nodes/stats/indices/fielddata/field1,field2?pretty
curl localhost:9200/_nodes/stats/indices/field1,field2?pretty
# Indices Stat
curl localhost:9200/_stats/fielddata/field1,field2?pretty
# You can use wildcards for field names
curl localhost:9200/_stats/fielddata/field*?pretty
curl localhost:9200/_nodes/stats/indices/fielddata/field*?pretty
curl localhost:9200/_nodes/stats/indices/field*?pretty
--------------------------------------------------
40 changes: 36 additions & 4 deletions src/main/java/org/elasticsearch/action/admin/indices/stats/IndexShardStats.java
View file
Expand Up @@ -20,19 +20,25 @@
package org.elasticsearch.action.admin.indices.stats;

import com.google.common.collect.Iterators;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.index.shard.ShardId;

import java.io.IOException;
import java.util.Iterator;

/**
*/
public class IndexShardStats implements Iterable<ShardStats> {
public class IndexShardStats implements Iterable<ShardStats>, Streamable {

private final ShardId shardId;
private ShardId shardId;

private final ShardStats[] shards;
private ShardStats[] shards;

IndexShardStats(ShardId shardId, ShardStats[] shards) {
private IndexShardStats() {}

public IndexShardStats(ShardId shardId, ShardStats[] shards) {
this.shardId = shardId;
this.shards = shards;
}
Expand Down Expand Up @@ -83,4 +89,30 @@ public CommonStats getPrimary() {
primary = stats;
return stats;
}

@Override
public void readFrom(StreamInput in) throws IOException {
shardId = ShardId.readShardId(in);
int shardSize = in.readVInt();
shards = new ShardStats[shardSize];
for (int i = 0; i < shardSize; i++) {
shards[i] = ShardStats.readShardStats(in);
}
}

@Override
public void writeTo(StreamOutput out) throws IOException {
shardId.writeTo(out);
out.writeVInt(shards.length);
for (ShardStats stats : shards) {
stats.writeTo(out);
}
}

public static IndexShardStats readIndexShardStats(StreamInput in) throws IOException {
IndexShardStats indexShardStats = new IndexShardStats();
indexShardStats.readFrom(in);
return indexShardStats;
}

}
31 changes: 19 additions & 12 deletions src/main/java/org/elasticsearch/indices/InternalIndicesService.java
View file
Expand Up @@ -19,14 +19,14 @@

package org.elasticsearch.indices;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.UnmodifiableIterator;
import com.google.common.collect.*;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.ElasticSearchIllegalStateException;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.inject.*;
Expand Down Expand Up @@ -72,6 +72,7 @@
import org.elasticsearch.plugins.PluginsService;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
Expand Down Expand Up @@ -175,44 +176,50 @@ public NodeIndicesStats stats(boolean includePrevious) {

@Override
public NodeIndicesStats stats(boolean includePrevious, CommonStatsFlags flags) {
CommonStats stats = new CommonStats(flags);
CommonStats oldStats = new CommonStats(flags);

if (includePrevious) {
Flag[] setFlags = flags.getFlags();
for (Flag flag : setFlags) {
switch (flag) {
case Get:
stats.get.add(oldShardsStats.getStats);
oldStats.get.add(oldShardsStats.getStats);
break;
case Indexing:
stats.indexing.add(oldShardsStats.indexingStats);
oldStats.indexing.add(oldShardsStats.indexingStats);
break;
case Search:
stats.search.add(oldShardsStats.searchStats);
oldStats.search.add(oldShardsStats.searchStats);
break;
case Merge:
stats.merge.add(oldShardsStats.mergeStats);
oldStats.merge.add(oldShardsStats.mergeStats);
break;
case Refresh:
stats.refresh.add(oldShardsStats.refreshStats);
oldStats.refresh.add(oldShardsStats.refreshStats);
break;
case Flush:
stats.flush.add(oldShardsStats.flushStats);
oldStats.flush.add(oldShardsStats.flushStats);
break;
}
}
}

Map<Index, List<IndexShardStats>> statsByShard = Maps.newHashMap();
for (IndexService indexService : indices.values()) {
for (IndexShard indexShard : indexService) {
try {
stats.add(new CommonStats(indexShard, flags));
IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard, flags) });
if (!statsByShard.containsKey(indexService.index())) {
statsByShard.put(indexService.index(), Lists.<IndexShardStats>newArrayList(indexShardStats));
} else {
statsByShard.get(indexService.index()).add(indexShardStats);
}
} catch (IllegalIndexShardStateException e) {
// we can safely ignore illegal state on ones that are closing for example
}
}
}
return new NodeIndicesStats(stats);
return new NodeIndicesStats(oldStats, statsByShard);
}

/**
Expand Down
99 changes: 97 additions & 2 deletions src/main/java/org/elasticsearch/indices/NodeIndicesStats.java
View file
Expand Up @@ -19,14 +19,19 @@

package org.elasticsearch.indices;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentBuilderString;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.cache.filter.FilterCacheStats;
import org.elasticsearch.index.cache.id.IdCacheStats;
import org.elasticsearch.index.engine.SegmentsStats;
Expand All @@ -39,24 +44,40 @@
import org.elasticsearch.index.refresh.RefreshStats;
import org.elasticsearch.index.search.stats.SearchStats;
import org.elasticsearch.index.shard.DocsStats;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.service.IndexShard;
import org.elasticsearch.index.store.StoreStats;
import org.elasticsearch.search.suggest.completion.CompletionStats;

import java.io.IOException;
import java.io.Serializable;
import java.util.List;
import java.util.Map;

/**
* Global information on indices stats running on a specific node.
*/
public class NodeIndicesStats implements Streamable, Serializable, ToXContent {

private CommonStats stats;
private Map<Index, List<IndexShardStats>> statsByShard;

NodeIndicesStats() {
}

public NodeIndicesStats(CommonStats stats) {
this.stats = stats;
public NodeIndicesStats(CommonStats oldStats, Map<Index, List<IndexShardStats>> statsByShard) {
//this.stats = stats;
this.statsByShard = statsByShard;

// make a total common stats from old ones and current ones
this.stats = oldStats;
for (List<IndexShardStats> shardStatsList : statsByShard.values()) {
for (IndexShardStats indexShardStats : shardStatsList) {
for (ShardStats shardStats : indexShardStats.getShards()) {
stats.add(shardStats.getStats());
}
}
}
}

@Nullable
Expand Down Expand Up @@ -138,21 +159,95 @@ public static NodeIndicesStats readIndicesStats(StreamInput in) throws IOExcepti
@Override
public void readFrom(StreamInput in) throws IOException {
stats = CommonStats.readCommonStats(in);
if (in.readBoolean()) {
int entries = in.readVInt();
statsByShard = Maps.newHashMap();
for (int i = 0; i < entries; i++) {
Index index = Index.readIndexName(in);
int indexShardListSize = in.readVInt();
List<IndexShardStats> indexShardStats = Lists.newArrayListWithCapacity(indexShardListSize);
for (int j = 0; j < indexShardListSize; j++) {
indexShardStats.add(IndexShardStats.readIndexShardStats(in));
}
statsByShard.put(index, indexShardStats);
}
}
}

@Override
public void writeTo(StreamOutput out) throws IOException {
stats.writeTo(out);
out.writeBoolean(statsByShard != null);
if (statsByShard != null) {
out.writeVInt(statsByShard.size());
for (Map.Entry<Index, List<IndexShardStats>> entry : statsByShard.entrySet()) {
entry.getKey().writeTo(out);
out.writeVInt(entry.getValue().size());
for (IndexShardStats indexShardStats : entry.getValue()) {
indexShardStats.writeTo(out);
}
}
}
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
String level = params.param("level", "node");
boolean isLevelValid = "node".equalsIgnoreCase(level) || "indices".equalsIgnoreCase(level) || "shards".equalsIgnoreCase(level);
if (!isLevelValid) {
return builder;
}

// "node" level
builder.startObject(Fields.INDICES);
stats.toXContent(builder, params);

if ("indices".equals(level)) {
Map<Index, CommonStats> indexStats = createStatsByIndex();
builder.startObject(Fields.INDICES);
for (Map.Entry<Index, CommonStats> entry : indexStats.entrySet()) {
builder.startObject(entry.getKey().name());
entry.getValue().toXContent(builder, params);
builder.endObject();
}
builder.endObject();
} else if ("shards".equals(level)) {
builder.startObject("shards");
for (Map.Entry<Index, List<IndexShardStats>> entry : statsByShard.entrySet()) {
builder.startArray(entry.getKey().name());
for (IndexShardStats indexShardStats : entry.getValue()) {
builder.startObject().startObject(String.valueOf(indexShardStats.getShardId().getId()));
for (ShardStats shardStats : indexShardStats.getShards()) {
shardStats.toXContent(builder, params);
}
builder.endObject().endObject();
}
builder.endArray();
}
builder.endObject();
}

builder.endObject();
return builder;
}

private Map<Index, CommonStats> createStatsByIndex() {
Map<Index, CommonStats> statsMap = Maps.newHashMap();
for (Map.Entry<Index, List<IndexShardStats>> entry : statsByShard.entrySet()) {
if (!statsMap.containsKey(entry.getKey())) {
statsMap.put(entry.getKey(), new CommonStats());
}

for (IndexShardStats indexShardStats : entry.getValue()) {
for (ShardStats shardStats : indexShardStats.getShards()) {
statsMap.get(entry.getKey()).add(shardStats.getStats());
}
}
}

return statsMap;
}

static final class Fields {
static final XContentBuilderString INDICES = new XContentBuilderString("indices");
}
Expand Down

0 comments on commit bb27516

Please sign in to comment.