Permalink
Browse files

Disabled query size estimation in percolator, because this is too exp…

…ensive cpu wise.

Lucene's RamUsageEstimator.sizeOf(Object) is to expensive.
Query size estimation will be enabled when a cheaper way of query size estimation can be found.

Closes #5372
Relates to #5339
  • Loading branch information...
martijnvg committed Mar 9, 2014
1 parent 7cad9b7 commit 7d84fd6d32167cf89babdec3347d044be44c523e
View
@@ -988,7 +988,6 @@
<exclude>org/elasticsearch/bootstrap/Bootstrap.class</exclude>
<exclude>org/elasticsearch/Version.class</exclude>
<exclude>org/apache/lucene/queries/XTermsFilter.class</exclude>
<exclude>org/elasticsearch/index/percolator/stats/ShardPercolateService$RamEstimator.class</exclude>
<exclude>org/elasticsearch/index/merge/Merges.class</exclude>
<!-- end excludes for valid system-out -->
<!-- start excludes for Unsafe -->
@@ -18,6 +18,7 @@
*/
package org.elasticsearch.index.percolator.stats;
import org.elasticsearch.Version;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
@@ -37,7 +38,7 @@
private long percolateCount;
private long percolateTimeInMillis;
private long current;
private long memorySizeInBytes;
private long memorySizeInBytes = -1;
private long numQueries;
/**
@@ -90,7 +91,9 @@ public long getNumQueries() {
}
/**
* @return The total size the loaded queries take in memory.
* @return Temporarily returns <code>-1</code>, but this used to return the total size the loaded queries take in
* memory, but this is disabled now because the size estimation was too expensive cpu wise. This will be enabled
* again when a cheaper size estimation can be found.
*/
public long getMemorySizeInBytes() {
return memorySizeInBytes;
@@ -124,7 +127,6 @@ public void add(PercolateStats percolate) {
percolateCount += percolate.getCount();
percolateTimeInMillis += percolate.getTimeInMillis();
current += percolate.getCurrent();
memorySizeInBytes += percolate.getMemorySizeInBytes();
numQueries += percolate.getNumQueries();
}
@@ -150,7 +152,11 @@ public void readFrom(StreamInput in) throws IOException {
percolateCount = in.readVLong();
percolateTimeInMillis = in.readVLong();
current = in.readVLong();
memorySizeInBytes = in.readVLong();
if (in.getVersion().before(Version.V_1_1_0)) {
in.readVLong();
} else {
in.readLong();
}
numQueries = in.readVLong();
}
@@ -159,7 +165,11 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeVLong(percolateCount);
out.writeVLong(percolateTimeInMillis);
out.writeVLong(current);
out.writeVLong(memorySizeInBytes);
if (out.getVersion().before(Version.V_1_1_0)) {
out.writeVLong(0);
} else {
out.writeLong(-1);
}
out.writeVLong(numQueries);
}
}
@@ -20,7 +20,6 @@
package org.elasticsearch.index.percolator.stats;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.RamUsageEstimator;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.HashedBytesRef;
import org.elasticsearch.common.metrics.CounterMetric;
@@ -38,7 +37,6 @@
* <li> total time spent in percolate api
* <li> the current number of percolate requests
* <li> number of registered percolate queries
* <li> the estimated amount of memory the registered queries take
* </ul>
*/
public class ShardPercolateService extends AbstractIndexShardComponent {
@@ -52,7 +50,6 @@ public ShardPercolateService(ShardId shardId, @IndexSettings Settings indexSetti
private final CounterMetric currentMetric = new CounterMetric();
private final CounterMetric numberOfQueries = new CounterMetric();
private final CounterMetric memorySizeInBytes = new CounterMetric();
public void prePercolate() {
currentMetric.inc();
@@ -64,27 +61,22 @@ public void postPercolate(long tookInNanos) {
}
public void addedQuery(HashedBytesRef id, Query previousQuery, Query newQuery) {
if (previousQuery != null) {
memorySizeInBytes.dec(computeSizeInMemory(id, previousQuery));
} else {
numberOfQueries.inc();
}
memorySizeInBytes.inc(computeSizeInMemory(id, newQuery));
numberOfQueries.inc();
}
public void removedQuery(HashedBytesRef id, Query query) {
numberOfQueries.dec();
memorySizeInBytes.dec(computeSizeInMemory(id, query));
}
/**
* @return The current metrics
*/
public PercolateStats stats() {
return new PercolateStats(percolateMetric.count(), TimeUnit.NANOSECONDS.toMillis(percolateMetric.sum()), currentMetric.count(), memorySizeInBytes.count(), numberOfQueries.count());
return new PercolateStats(percolateMetric.count(), TimeUnit.NANOSECONDS.toMillis(percolateMetric.sum()), currentMetric.count(), -1, numberOfQueries.count());
}
private static long computeSizeInMemory(HashedBytesRef id, Query query) {
// Enable when a more efficient manner is found for estimating the size of a Lucene query.
/*private static long computeSizeInMemory(HashedBytesRef id, Query query) {
long size = (3 * RamUsageEstimator.NUM_BYTES_INT) + RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + id.bytes.bytes.length;
size += RamEstimator.sizeOf(query);
return size;
@@ -96,6 +88,6 @@ private static long computeSizeInMemory(HashedBytesRef id, Query query) {
static long sizeOf(Query query) {
return RamUsageEstimator.sizeOf(query);
}
}
}*/
}
@@ -558,7 +558,7 @@ public void testPercolateStatistics() throws Exception {
assertThat(indicesResponse.getTotal().getPercolate().getCount(), equalTo((long) numShards.numPrimaries));
assertThat(indicesResponse.getTotal().getPercolate().getCurrent(), equalTo(0l));
assertThat(indicesResponse.getTotal().getPercolate().getNumQueries(), equalTo((long)numShards.dataCopies)); //number of copies
assertThat(indicesResponse.getTotal().getPercolate().getMemorySizeInBytes(), greaterThan(0l));
assertThat(indicesResponse.getTotal().getPercolate().getMemorySizeInBytes(), equalTo(-1l));
NodesStatsResponse nodesResponse = client().admin().cluster().prepareNodesStats().execute().actionGet();
long percolateCount = 0;
@@ -580,7 +580,7 @@ public void testPercolateStatistics() throws Exception {
assertThat(indicesResponse.getTotal().getPercolate().getCount(), equalTo((long) numShards.numPrimaries * 2));
assertThat(indicesResponse.getTotal().getPercolate().getCurrent(), equalTo(0l));
assertThat(indicesResponse.getTotal().getPercolate().getNumQueries(), equalTo((long)numShards.dataCopies)); //number of copies
assertThat(indicesResponse.getTotal().getPercolate().getMemorySizeInBytes(), greaterThan(0l));
assertThat(indicesResponse.getTotal().getPercolate().getMemorySizeInBytes(), equalTo(-1l));
percolateCount = 0;
nodesResponse = client().admin().cluster().prepareNodesStats().execute().actionGet();

0 comments on commit 7d84fd6

Please sign in to comment.