Skip to content

Commit

Permalink
Translog: stats fail to serialize size
Browse files Browse the repository at this point in the history 
Closes #10105
  • Loading branch information
@bleskes
bleskes committed Mar 16, 2015
1 parent 1646638 commit acd095d
Show file tree
Hide file tree
Showing 2 changed files with 96 additions and 25 deletions.
18 changes: 18 additions & 0 deletions src/main/java/org/elasticsearch/index/translog/TranslogStats.java
View file
Expand Up @@ -18,9 +18,11 @@
*/
package org.elasticsearch.index.translog;

import org.elasticsearch.Version;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentBuilderString;
Expand All @@ -38,6 +40,8 @@ public class TranslogStats implements ToXContent, Streamable {
public TranslogStats() {}

public TranslogStats(int estimatedNumberOfOperations, long translogSizeInBytes) {
assert estimatedNumberOfOperations >= 0 : "estimatedNumberOfOperations must be >=0, got [" + estimatedNumberOfOperations + "]";
assert translogSizeInBytes >= 0 : "translogSizeInBytes must be >=0, got [" + translogSizeInBytes + "]";
this.estimatedNumberOfOperations = estimatedNumberOfOperations;
this.translogSizeInBytes = translogSizeInBytes;
}
Expand All @@ -51,6 +55,14 @@ public void add(TranslogStats translogStats) {
this.translogSizeInBytes =+ translogStats.translogSizeInBytes;
}

public ByteSizeValue translogSizeInBytes() {
return new ByteSizeValue(translogSizeInBytes);
}

public long estimatedNumberOfOperations() {
return estimatedNumberOfOperations;
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(Fields.TRANSLOG);
Expand All @@ -70,10 +82,16 @@ static final class Fields {
@Override
public void readFrom(StreamInput in) throws IOException {
estimatedNumberOfOperations = in.readVInt();
if (in.getVersion().onOrAfter(Version.V_1_4_5)) {
translogSizeInBytes = in.readVLong();
}
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(estimatedNumberOfOperations);
if (out.getVersion().onOrAfter(Version.V_1_4_5)) {
out.writeVLong(translogSizeInBytes);
}
}
}
103 changes: 78 additions & 25 deletions src/test/java/org/elasticsearch/index/translog/AbstractSimpleTranslogTests.java
View file
Expand Up @@ -23,18 +23,19 @@
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.io.stream.BytesStreamInput;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.VersionType;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.hamcrest.MatcherAssert;
import org.hamcrest.Matchers;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.File;
import java.io.EOFException;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.file.Files;
Expand All @@ -47,8 +48,7 @@
import java.util.concurrent.atomic.AtomicInteger;

import static com.google.common.collect.Lists.newArrayList;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.*;

/**
*
Expand Down Expand Up @@ -94,63 +94,63 @@ public void testRead() throws IOException {
@Test
public void testTransientTranslog() {
Translog.Snapshot snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
snapshot.close();

translog.add(new Translog.Create("test", "1", new byte[]{1}));
snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot.estimatedTotalOperations(), equalTo(1));
snapshot.close();

translog.newTransientTranslog(2);

snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot.estimatedTotalOperations(), equalTo(1));
snapshot.close();

translog.add(new Translog.Index("test", "2", new byte[]{2}));
snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(2));
assertThat(snapshot, TranslogSizeMatcher.translogSize(2));
assertThat(snapshot.estimatedTotalOperations(), equalTo(2));
snapshot.close();

translog.makeTransientCurrent();

snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1)); // now its one, since it only includes "2"
assertThat(snapshot, TranslogSizeMatcher.translogSize(1)); // now its one, since it only includes "2"
assertThat(snapshot.estimatedTotalOperations(), equalTo(1));
snapshot.close();
}

@Test
public void testSimpleOperations() {
Translog.Snapshot snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
snapshot.close();

translog.add(new Translog.Create("test", "1", new byte[]{1}));
snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot.estimatedTotalOperations(), equalTo(1));
snapshot.close();

translog.add(new Translog.Index("test", "2", new byte[]{2}));
snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(2));
assertThat(snapshot, TranslogSizeMatcher.translogSize(2));
assertThat(snapshot.estimatedTotalOperations(), equalTo(2));
snapshot.close();

translog.add(new Translog.Delete(newUid("3")));
snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(3));
assertThat(snapshot, TranslogSizeMatcher.translogSize(3));
assertThat(snapshot.estimatedTotalOperations(), equalTo(3));
snapshot.close();

translog.add(new Translog.DeleteByQuery(new BytesArray(new byte[]{4}), null));
snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(4));
assertThat(snapshot, TranslogSizeMatcher.translogSize(4));
assertThat(snapshot.estimatedTotalOperations(), equalTo(4));
snapshot.close();

Expand Down Expand Up @@ -181,20 +181,73 @@ public void testSimpleOperations() {
assertThat(translog.currentId(), Matchers.not(equalTo(firstId)));

snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
assertThat(snapshot.estimatedTotalOperations(), equalTo(0));
snapshot.close();
}


protected TranslogStats stats() throws IOException {
// force flushing and updating of stats
translog.sync();
TranslogStats stats = translog.stats();
if (randomBoolean()) {
BytesStreamOutput out = new BytesStreamOutput();
stats.writeTo(out);
BytesStreamInput in = new BytesStreamInput(out.bytes());
stats = new TranslogStats();
stats.readFrom(in);
}
return stats;
}

@Test
public void testStats() throws IOException {
TranslogStats stats = stats();
assertThat(stats.estimatedNumberOfOperations(), equalTo(0l));
long lastSize = stats.translogSizeInBytes().bytes();
assertThat(lastSize, equalTo(17l));

translog.add(new Translog.Create("test", "1", new byte[]{1}));
stats = stats();
assertThat(stats.estimatedNumberOfOperations(), equalTo(1l));
assertThat(stats.translogSizeInBytes().bytes(), greaterThan(lastSize));
lastSize = stats.translogSizeInBytes().bytes();

translog.add(new Translog.Index("test", "2", new byte[]{2}));
stats = stats();
assertThat(stats.estimatedNumberOfOperations(), equalTo(2l));
assertThat(stats.translogSizeInBytes().bytes(), greaterThan(lastSize));
lastSize = stats.translogSizeInBytes().bytes();

translog.add(new Translog.Delete(newUid("3")));
stats = stats();
assertThat(stats.estimatedNumberOfOperations(), equalTo(3l));
assertThat(stats.translogSizeInBytes().bytes(), greaterThan(lastSize));
lastSize = stats.translogSizeInBytes().bytes();


translog.add(new Translog.DeleteByQuery(new BytesArray(new byte[]{4}), null));
stats = stats();
assertThat(stats.estimatedNumberOfOperations(), equalTo(4l));
assertThat(stats.translogSizeInBytes().bytes(), greaterThan(lastSize));

translog.newTranslog(2);
stats = stats();
assertThat(stats.estimatedNumberOfOperations(), equalTo(0l));
assertThat(stats.translogSizeInBytes().bytes(), equalTo(17l));
}


@Test
public void testSnapshot() {
Translog.Snapshot snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
snapshot.close();

translog.add(new Translog.Create("test", "1", new byte[]{1}));
snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot.estimatedTotalOperations(), equalTo(1));
snapshot.close();

Expand All @@ -205,7 +258,7 @@ public void testSnapshot() {
snapshot.close();

Translog.Snapshot snapshot1 = translog.snapshot();
MatcherAssert.assertThat(snapshot1, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot1, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot1.estimatedTotalOperations(), equalTo(1));

// seek to the end of the translog snapshot
Expand All @@ -215,7 +268,7 @@ public void testSnapshot() {

translog.add(new Translog.Index("test", "2", new byte[]{2}));
snapshot = translog.snapshot(snapshot1);
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot.estimatedTotalOperations(), equalTo(2));
snapshot.close();

Expand All @@ -232,7 +285,7 @@ public void testSnapshot() {
@Test
public void testSnapshotWithNewTranslog() {
Translog.Snapshot snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
snapshot.close();

translog.add(new Translog.Create("test", "1", new byte[]{1}));
Expand All @@ -245,7 +298,7 @@ public void testSnapshotWithNewTranslog() {
translog.add(new Translog.Index("test", "3", new byte[]{3}));

snapshot = translog.snapshot(actualSnapshot);
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
snapshot.close();

snapshot = translog.snapshot(actualSnapshot);
Expand All @@ -261,12 +314,12 @@ public void testSnapshotWithNewTranslog() {
@Test
public void testSnapshotWithSeekTo() {
Translog.Snapshot snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
snapshot.close();

translog.add(new Translog.Create("test", "1", new byte[]{1}));
snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
// seek to the end of the translog snapshot
while (snapshot.next() != null) {
// spin
Expand All @@ -277,7 +330,7 @@ public void testSnapshotWithSeekTo() {
translog.add(new Translog.Create("test", "2", new byte[]{1}));
snapshot = translog.snapshot();
snapshot.seekTo(lastPosition);
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
snapshot.close();

snapshot = translog.snapshot();
Expand Down Expand Up @@ -516,7 +569,7 @@ public void testVerifyTranslogIsNotDeletedBySnapshotClose() throws IOException {
assertTrue(Files.exists(path.resolve("translog-1")));
translog.add(new Translog.Create("test", "1", new byte[]{1}));
Translog.Snapshot snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot.estimatedTotalOperations(), equalTo(1));
if (randomBoolean()) {
translog.close();
Expand Down

0 comments on commit acd095d

Please sign in to comment.