Skip to content

Commit

Permalink
[STORE] Calculate Alder32 Checksums for legacy files in Store#checkIn…
Browse files Browse the repository at this point in the history 
…tegrity

Previously we didn't calculate this checksums even though we have a checksum
to compare. Since we now also verify checksums for legacy files #checkIntegrity
should also calculate the legacy checksums.

Closes elastic#8407
  • Loading branch information
@s1monw
s1monw committed Nov 9, 2014
1 parent f6279a7 commit 4c5e2f5
Show file tree
Hide file tree
Showing 2 changed files with 104 additions and 4 deletions.
32 changes: 28 additions & 4 deletions src/main/java/org/elasticsearch/index/store/Store.java
View file
Expand Up @@ -52,6 +52,9 @@
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.Adler32;
import java.util.zip.CRC32;
import java.util.zip.Checksum;

/**
* A Store provides plain access to files written by an elasticsearch index shard. Each shard
Expand Down Expand Up @@ -364,12 +367,33 @@ public static void verify(IndexOutput output) throws IOException {
}

public boolean checkIntegrity(StoreFileMetaData md) {
if (md.writtenBy() != null && md.writtenBy().onOrAfter(Version.LUCENE_48)) {
try (IndexInput input = directory().openInput(md.name(), IOContext.READONCE)) {
CodecUtil.checksumEntireFile(input);
} catch (IOException e) {
return checkIntegrity(md, directory());
}

public static boolean checkIntegrity(final StoreFileMetaData md, final Directory directory) {
try (IndexInput input = directory.openInput(md.name(), IOContext.READONCE)) {
if (input.length() != md.length()) { // first check the length no matter how old this file is
return false;
}
if (md.writtenBy() != null && md.writtenBy().onOrAfter(Version.LUCENE_48)) {
return Store.digestToString(CodecUtil.checksumEntireFile(input)).equals(md.checksum());
} else if (md.hasLegacyChecksum()) {
// legacy checksum verification - no footer that we need to omit in the checksum!
final Checksum checksum = new Adler32();
final byte[] buffer = new byte[md.length() > 4096 ? 4096 : (int) md.length()];
final long len = input.length();
long read = 0;
while (len > read) {
final long bytesLeft = len - read;
final int bytesToRead = bytesLeft < buffer.length ? (int) bytesLeft : buffer.length;
input.readBytes(buffer, 0, bytesToRead, false);
checksum.update(buffer, 0, bytesToRead);
read += bytesToRead;
}
return Store.digestToString(checksum.getValue()).equals(md.checksum());
}
} catch (IOException ex) {
return false;
}
return true;
}
Expand Down
76 changes: 76 additions & 0 deletions src/test/java/org/elasticsearch/index/store/StoreTest.java
View file
Expand Up @@ -18,6 +18,7 @@
*/
package org.elasticsearch.index.store;

import com.carrotsearch.randomizedtesting.annotations.Repeat;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.document.*;
Expand All @@ -26,6 +27,7 @@
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.Version;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.shard.ShardId;
Expand All @@ -39,6 +41,7 @@
import java.io.IOException;
import java.nio.file.NoSuchFileException;
import java.util.*;
import java.util.zip.Adler32;

import static org.hamcrest.Matchers.*;

Expand Down Expand Up @@ -374,6 +377,79 @@ public void testRenameFile() throws IOException {
IOUtils.close(store);
}

public void testCheckIntegrity() throws IOException {
Directory dir = newDirectory();
long luceneFileLength = 0;

try (IndexOutput output = dir.createOutput("lucene_checksum.bin", IOContext.DEFAULT)) {
int iters = scaledRandomIntBetween(10, 100);
for (int i = 0; i < iters; i++) {
BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
luceneFileLength += bytesRef.length;
}
CodecUtil.writeFooter(output);
luceneFileLength += CodecUtil.footerLength();

}

final Adler32 adler32 = new Adler32();
long legacyFileLength = 0;
try (IndexOutput output = dir.createOutput("legacy.bin", IOContext.DEFAULT)) {
int iters = scaledRandomIntBetween(10, 100);
for (int i = 0; i < iters; i++) {
BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
adler32.update(bytesRef.bytes, bytesRef.offset, bytesRef.length);
legacyFileLength += bytesRef.length;
}
}
final long luceneChecksum;
final long adler32LegacyChecksum = adler32.getValue();
try(IndexInput indexInput = dir.openInput("lucene_checksum.bin", IOContext.DEFAULT)) {
assertEquals(luceneFileLength, indexInput.length());
luceneChecksum = CodecUtil.retrieveChecksum(indexInput);
}

{ // positive check
StoreFileMetaData lucene = new StoreFileMetaData("lucene_checksum.bin", luceneFileLength, Store.digestToString(luceneChecksum), Version.LUCENE_48);
StoreFileMetaData legacy = new StoreFileMetaData("legacy.bin", legacyFileLength, Store.digestToString(adler32LegacyChecksum));
assertTrue(legacy.hasLegacyChecksum());
assertFalse(lucene.hasLegacyChecksum());
assertTrue(Store.checkIntegrity(lucene, dir));
assertTrue(Store.checkIntegrity(legacy, dir));
}

{ // negative check - wrong checksum
StoreFileMetaData lucene = new StoreFileMetaData("lucene_checksum.bin", luceneFileLength, Store.digestToString(luceneChecksum+1), Version.LUCENE_48);
StoreFileMetaData legacy = new StoreFileMetaData("legacy.bin", legacyFileLength, Store.digestToString(adler32LegacyChecksum+1));
assertTrue(legacy.hasLegacyChecksum());
assertFalse(lucene.hasLegacyChecksum());
assertFalse(Store.checkIntegrity(lucene, dir));
assertFalse(Store.checkIntegrity(legacy, dir));
}

{ // negative check - wrong length
StoreFileMetaData lucene = new StoreFileMetaData("lucene_checksum.bin", luceneFileLength+1, Store.digestToString(luceneChecksum), Version.LUCENE_48);
StoreFileMetaData legacy = new StoreFileMetaData("legacy.bin", legacyFileLength+1, Store.digestToString(adler32LegacyChecksum));
assertTrue(legacy.hasLegacyChecksum());
assertFalse(lucene.hasLegacyChecksum());
assertFalse(Store.checkIntegrity(lucene, dir));
assertFalse(Store.checkIntegrity(legacy, dir));
}

{ // negative check - wrong file
StoreFileMetaData lucene = new StoreFileMetaData("legacy.bin", luceneFileLength, Store.digestToString(luceneChecksum), Version.LUCENE_48);
StoreFileMetaData legacy = new StoreFileMetaData("lucene_checksum.bin", legacyFileLength, Store.digestToString(adler32LegacyChecksum));
assertTrue(legacy.hasLegacyChecksum());
assertFalse(lucene.hasLegacyChecksum());
assertFalse(Store.checkIntegrity(lucene, dir));
assertFalse(Store.checkIntegrity(legacy, dir));
}
dir.close();

}

public void assertDeleteContent(Store store,DirectoryService service) throws IOException {
store.deleteContent();
assertThat(Arrays.toString(store.directory().listAll()), store.directory().listAll().length, equalTo(0));
Expand Down

0 comments on commit 4c5e2f5

Please sign in to comment.