New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Disabled query size estimation in percolator #5372

Closed
wants to merge 1 commit into
base: master
from
Jump to file or symbol
Failed to load files and symbols.
+22 −21
Diff settings

Always

Just for now

Disabled query size estimation in percolator, because this is too exp…

…ensive cpu wise.

Lucene's RamUsageEstimator.sizeOf(Object) is to expensive.
Query size estimation will be enabled when a cheaper way of query size estimation can be found.

Closes #5372
Relates to #5339
  • Loading branch information...
martijnvg committed Mar 9, 2014
commit 8b75c240b57b711229fd0dccac3a22aeabdebab5
Copy path View file
@@ -988,7 +988,6 @@
<exclude>org/elasticsearch/bootstrap/Bootstrap.class</exclude>
<exclude>org/elasticsearch/Version.class</exclude>
<exclude>org/apache/lucene/queries/XTermsFilter.class</exclude>
<exclude>org/elasticsearch/index/percolator/stats/ShardPercolateService$RamEstimator.class</exclude>
<exclude>org/elasticsearch/index/merge/Merges.class</exclude>
<!-- end excludes for valid system-out -->
<!-- start excludes for Unsafe -->
@@ -18,6 +18,7 @@
*/
package org.elasticsearch.index.percolator.stats;
import org.elasticsearch.Version;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
@@ -37,7 +38,7 @@
private long percolateCount;
private long percolateTimeInMillis;
private long current;
private long memorySizeInBytes;
private long memorySizeInBytes = -1;
private long numQueries;
/**
@@ -90,7 +91,9 @@ public long getNumQueries() {
}
/**
* @return The total size the loaded queries take in memory.
* @return Temporarily returns <code>-1</code>, but this used to return the total size the loaded queries take in
* memory, but this is disabled now because the size estimation was too expensive cpu wise. This will be enabled
* again when a cheaper size estimation can be found.
*/
public long getMemorySizeInBytes() {
return memorySizeInBytes;
@@ -124,7 +127,6 @@ public void add(PercolateStats percolate) {
percolateCount += percolate.getCount();
percolateTimeInMillis += percolate.getTimeInMillis();
current += percolate.getCurrent();
memorySizeInBytes += percolate.getMemorySizeInBytes();
numQueries += percolate.getNumQueries();
}
@@ -150,7 +152,11 @@ public void readFrom(StreamInput in) throws IOException {
percolateCount = in.readVLong();
percolateTimeInMillis = in.readVLong();
current = in.readVLong();
memorySizeInBytes = in.readVLong();
if (in.getVersion().before(Version.V_1_1_0)) {
in.readVLong();
} else {
in.readLong();
}
numQueries = in.readVLong();
}
@@ -159,7 +165,11 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeVLong(percolateCount);
out.writeVLong(percolateTimeInMillis);
out.writeVLong(current);
out.writeVLong(memorySizeInBytes);
if (out.getVersion().before(Version.V_1_1_0)) {
out.writeVLong(0);
} else {
out.writeLong(-1);
}
out.writeVLong(numQueries);
}
}
@@ -20,7 +20,6 @@
package org.elasticsearch.index.percolator.stats;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.RamUsageEstimator;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.HashedBytesRef;
import org.elasticsearch.common.metrics.CounterMetric;
@@ -38,7 +37,6 @@
* <li> total time spent in percolate api
* <li> the current number of percolate requests
* <li> number of registered percolate queries
* <li> the estimated amount of memory the registered queries take
* </ul>
*/
public class ShardPercolateService extends AbstractIndexShardComponent {
@@ -52,7 +50,6 @@ public ShardPercolateService(ShardId shardId, @IndexSettings Settings indexSetti
private final CounterMetric currentMetric = new CounterMetric();
private final CounterMetric numberOfQueries = new CounterMetric();
private final CounterMetric memorySizeInBytes = new CounterMetric();
public void prePercolate() {
currentMetric.inc();
@@ -64,27 +61,22 @@ public void postPercolate(long tookInNanos) {
}
public void addedQuery(HashedBytesRef id, Query previousQuery, Query newQuery) {
if (previousQuery != null) {
memorySizeInBytes.dec(computeSizeInMemory(id, previousQuery));
} else {
numberOfQueries.inc();
}
memorySizeInBytes.inc(computeSizeInMemory(id, newQuery));
numberOfQueries.inc();
}
public void removedQuery(HashedBytesRef id, Query query) {
numberOfQueries.dec();
memorySizeInBytes.dec(computeSizeInMemory(id, query));
}
/**
* @return The current metrics
*/
public PercolateStats stats() {
return new PercolateStats(percolateMetric.count(), TimeUnit.NANOSECONDS.toMillis(percolateMetric.sum()), currentMetric.count(), memorySizeInBytes.count(), numberOfQueries.count());
return new PercolateStats(percolateMetric.count(), TimeUnit.NANOSECONDS.toMillis(percolateMetric.sum()), currentMetric.count(), -1, numberOfQueries.count());
}
private static long computeSizeInMemory(HashedBytesRef id, Query query) {
// Enable when a more efficient manner is found for estimating the size of a Lucene query.
/*private static long computeSizeInMemory(HashedBytesRef id, Query query) {
long size = (3 * RamUsageEstimator.NUM_BYTES_INT) + RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + id.bytes.bytes.length;
size += RamEstimator.sizeOf(query);
return size;
@@ -96,6 +88,6 @@ private static long computeSizeInMemory(HashedBytesRef id, Query query) {
static long sizeOf(Query query) {
return RamUsageEstimator.sizeOf(query);
}
}
}*/
}
@@ -558,7 +558,7 @@ public void testPercolateStatistics() throws Exception {
assertThat(indicesResponse.getTotal().getPercolate().getCount(), equalTo((long) numShards.numPrimaries));
assertThat(indicesResponse.getTotal().getPercolate().getCurrent(), equalTo(0l));
assertThat(indicesResponse.getTotal().getPercolate().getNumQueries(), equalTo((long)numShards.dataCopies)); //number of copies
assertThat(indicesResponse.getTotal().getPercolate().getMemorySizeInBytes(), greaterThan(0l));
assertThat(indicesResponse.getTotal().getPercolate().getMemorySizeInBytes(), equalTo(-1l));
NodesStatsResponse nodesResponse = client().admin().cluster().prepareNodesStats().execute().actionGet();
long percolateCount = 0;
@@ -580,7 +580,7 @@ public void testPercolateStatistics() throws Exception {
assertThat(indicesResponse.getTotal().getPercolate().getCount(), equalTo((long) numShards.numPrimaries * 2));
assertThat(indicesResponse.getTotal().getPercolate().getCurrent(), equalTo(0l));
assertThat(indicesResponse.getTotal().getPercolate().getNumQueries(), equalTo((long)numShards.dataCopies)); //number of copies
assertThat(indicesResponse.getTotal().getPercolate().getMemorySizeInBytes(), greaterThan(0l));
assertThat(indicesResponse.getTotal().getPercolate().getMemorySizeInBytes(), equalTo(-1l));
percolateCount = 0;
nodesResponse = client().admin().cluster().prepareNodesStats().execute().actionGet();
ProTip! Use n and p to navigate between commits in a pull request.