Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Disable RAM usage estimation on Lucene 3.x segments. #5202

Closed
wants to merge 1 commit into from
Closed

Disable RAM usage estimation on Lucene 3.x segments. #5202

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
18 changes: 15 additions & 3 deletions src/main/java/org/elasticsearch/index/engine/SegmentsStats.java
View file
Open in desktop
Expand Up @@ -19,9 +19,11 @@

package org.elasticsearch.index.engine;

import org.apache.lucene.util.Version;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
Expand All @@ -41,17 +43,27 @@ public SegmentsStats() {

}

static {
assert Version.LUCENE_46.onOrAfter(Lucene.VERSION); // remove special -1 handling below
}

public void add(long count, long memoryInBytes) {
this.count += count;
this.memoryInBytes += memoryInBytes;
// replace me with
// "this.memoryInBytes += memoryInBytes;"
// upon upgrade to Lucene 4.7
if (memoryInBytes == -1) {
this.memoryInBytes = -1;
} else if (this.memoryInBytes != -1) {
this.memoryInBytes += memoryInBytes;
}
}

public void add(SegmentsStats mergeStats) {
if (mergeStats == null) {
return;
}
this.count += mergeStats.count;
this.memoryInBytes += mergeStats.memoryInBytes;
add(mergeStats.count, mergeStats.memoryInBytes);
}

/**
Expand Down
15 changes: 13 additions & 2 deletions src/main/java/org/elasticsearch/index/engine/internal/InternalEngine.java
View file
Open in desktop
Expand Up @@ -20,6 +20,7 @@
package org.elasticsearch.index.engine.internal;

import com.google.common.collect.Lists;
import org.apache.lucene.codecs.lucene3x.Lucene3xCodec;
import org.apache.lucene.index.*;
import org.apache.lucene.index.IndexWriter.IndexReaderWarmer;
import org.apache.lucene.search.IndexSearcher;
Expand All @@ -30,6 +31,7 @@
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.cluster.routing.operation.hash.djb.DjbHashFunction;
Expand Down Expand Up @@ -1118,9 +1120,18 @@ public void recover(RecoveryHandler recoveryHandler) throws EngineException {
}
}

static {
assert Version.LUCENE_46.onOrAfter(Lucene.VERSION); // Lucene 4.7 fixed Lucene 3.X RAM usage estimations, see LUCENE-5462
}

private long getReaderRamBytesUsed(AtomicReaderContext reader) {
return SegmentReaderUtils.segmentReader(reader.reader()).ramBytesUsed();
final SegmentReader segmentReader = SegmentReaderUtils.segmentReader(reader.reader());
if (segmentReader.getSegmentInfo().info.getCodec() instanceof Lucene3xCodec) {
// https://issues.apache.org/jira/browse/LUCENE-5462
// RAM usage estimation is very costly on Lucene 3.x segments
return -1;
}
return segmentReader.ramBytesUsed();
}

@Override
Expand Down Expand Up @@ -1357,7 +1368,7 @@ private IndexWriter createWriter() throws IOException {
config.setMaxThreadStates(indexConcurrency);
config.setCodec(codecService.codec(codecName));
/* We set this timeout to a highish value to work around
* the default poll interval in the Lucene lock that is
* the default poll interval in the Lucene lock that is
* 1000ms by default. We might need to poll multiple times
* here but with 1s poll this is only executed twice at most
* in combination with the default writelock timeout*/
Expand Down