Skip to content

Commit

Permalink
Don't verify adler32 for buggy legacy checksums.
Browse files Browse the repository at this point in the history 
Don't verify adler32 for lucene 3.x terms dictionary (.tis),
terms index (.tii), 3.0-3.3 compound file (.cfs), 3.x segments_N,
or 3.x segments.gen.

These files are not write-once, but the 0.20.x checksum code
didn't handle .tis/.tii properly, so we can't use the adler32.
Even older versions will have the same problems with other cases.

Closes elastic#9142

Conflicts:
	src/main/java/org/elasticsearch/index/store/Store.java
	src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityTests.java
  • Loading branch information
@rmuir
rmuir committed Jan 6, 2015
1 parent 455c365 commit 0ad2a4a
Show file tree
Hide file tree
Showing 2 changed files with 167 additions and 5 deletions.
25 changes: 21 additions & 4 deletions src/main/java/org/elasticsearch/index/store/Store.java
View file
Expand Up @@ -329,17 +329,34 @@ public static MetadataSnapshot readMetadataSnapshot(File[] indexLocations, ESLog
* The returned IndexOutput might validate the files checksum if the file has been written with a newer lucene version
* and the metadata holds the necessary information to detect that it was been written by Lucene 4.8 or newer. If it has only
* a legacy checksum, returned IndexOutput will not verify the checksum.
*
* Note: Checksums are calculated nevertheless since lucene does it by default sicne version 4.8.0. This method only adds the
* <p/>
* Note: Checksums are calculated nevertheless since lucene does it by default since version 4.8.0. This method only adds the
* verification against the checksum in the given metadata and does not add any significant overhead.
*/
public IndexOutput createVerifyingOutput(String fileName, final StoreFileMetaData metadata, final IOContext context) throws IOException {
IndexOutput output = directory().createOutput(fileName, context);
boolean success = false;
try {
if (metadata.hasLegacyChecksum()) {
logger.debug("create legacy adler32 output for {}", fileName);
output = new LegacyVerification.Adler32VerifyingIndexOutput(output, metadata.checksum(), metadata.length());
// Lucene's .tii and .tis files (3.x) were not actually append-only.
// this means the adler32 in ES 0.20.x releases is actually wrong, we can't use it.
boolean badTermInfo = metadata.name().endsWith(".tii") || metadata.name().endsWith(".tis");
// Lucene's .cfs (3.0-3.3) was not append-only, so old ES checksums (pre-0.18.x) are wrong.
// NOTE: if we don't know the version, then we don't trust the checksum.
boolean badCFS = metadata.name().endsWith(".cfs") &&
(metadata.writtenBy() == null || metadata.writtenBy().onOrAfter(Version.LUCENE_34) == false);
// Lucene's segments_N always had a checksum, and reasonably old versions of ES never added
// their own metadata for it. But it wasn't append-only, so be defensive and don't rely upon
// old versions omitting the checksum. NOTE: This logic also excludes segments.gen for the same reason.
boolean badCommit = metadata.name().startsWith(IndexFileNames.SEGMENTS);

if (badTermInfo || badCFS || badCommit) {
logger.debug("create legacy length-only output for non-write-once file {}", fileName);
output = new LegacyVerification.LengthVerifyingIndexOutput(output, metadata.length());
} else {
logger.debug("create legacy adler32 output for {}", fileName);
output = new LegacyVerification.Adler32VerifyingIndexOutput(output, metadata.checksum(), metadata.length());
}
} else if (metadata.checksum() == null) {
// TODO: when the file is a segments_N, we can still CRC-32 + length for more safety
// its had that checksum forever.
Expand Down
147 changes: 146 additions & 1 deletion src/test/java/org/elasticsearch/index/store/StoreTest.java
View file
Expand Up @@ -18,7 +18,6 @@
*/
package org.elasticsearch.index.store;

import com.carrotsearch.randomizedtesting.annotations.Repeat;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.document.*;
Expand Down Expand Up @@ -377,6 +376,152 @@ public void testMixedChecksums() throws IOException {
IOUtils.close(store);
}

// Test cases with incorrect adler32 in their metadata caused by old versions.

@Test
public void testBuggyTIIChecksums() throws IOException {
final ShardId shardId = new ShardId(new Index("index"), 1);
DirectoryService directoryService = new LuceneManagedDirectoryService(random(), false);
Store store = new Store(shardId, ImmutableSettings.EMPTY, null, directoryService, randomDistributor(directoryService));

// .tii: no version specified
StoreFileMetaData tii = new StoreFileMetaData("foo.tii", 20, "boguschecksum", null);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp", tii, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
}

// .tii: old version
tii = new StoreFileMetaData("foo.tii", 20, "boguschecksum", Version.LUCENE_36);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp2", tii, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
}
store.close();
}

@Test
public void testBuggyTISChecksums() throws IOException {
final ShardId shardId = new ShardId(new Index("index"), 1);
DirectoryService directoryService = new LuceneManagedDirectoryService(random(), false);
Store store = new Store(shardId, ImmutableSettings.EMPTY, null, directoryService, randomDistributor(directoryService));

// .tis: no version specified
StoreFileMetaData tis = new StoreFileMetaData("foo.tis", 20, "boguschecksum", null);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp", tis, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
};

// .tis: old version
tis = new StoreFileMetaData("foo.tis", 20, "boguschecksum", Version.LUCENE_36);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp", tis, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
};

store.close();
}

@Test
public void testBuggyCFSChecksums() throws IOException {
final ShardId shardId = new ShardId(new Index("index"), 1);
DirectoryService directoryService = new LuceneManagedDirectoryService(random(), false);
Store store = new Store(shardId, ImmutableSettings.EMPTY, null, directoryService, randomDistributor(directoryService));

// .cfs: unspecified version
StoreFileMetaData cfs = new StoreFileMetaData("foo.cfs", 20, "boguschecksum", null);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp", cfs, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
}

// .cfs: ancient affected version
cfs = new StoreFileMetaData("foo.cfs", 20, "boguschecksum", Version.LUCENE_33);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp2", cfs, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
}

// .cfs: should still be checksummed for an ok version
cfs = new StoreFileMetaData("foo.cfs", 20, "boguschecksum", Version.LUCENE_34);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp3", cfs, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
fail("should have gotten expected exception");
} catch (CorruptIndexException expected) {
assertTrue(expected.getMessage().startsWith("checksum failed"));
}

store.close();
}

@Test
public void testSegmentsNChecksums() throws IOException {
final ShardId shardId = new ShardId(new Index("index"), 1);
DirectoryService directoryService = new LuceneManagedDirectoryService(random(), false);
Store store = new Store(shardId, ImmutableSettings.EMPTY, null, directoryService, randomDistributor(directoryService));

// segments_N: unspecified version
StoreFileMetaData segments = new StoreFileMetaData("segments_1", 20, "boguschecksum", null);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp", segments, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
}

// segments_N: specified old version
segments = new StoreFileMetaData("segments_2", 20, "boguschecksum", Version.LUCENE_33);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp2", segments, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
}

// segments_N: should still be checksummed for an ok version (lucene checksum)
segments = new StoreFileMetaData("segments_3", 20, "boguschecksum", Version.LUCENE_48);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp3", segments, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
fail("should have gotten expected exception");
} catch (CorruptIndexException expected) {
assertTrue(expected.getMessage().startsWith("checksum failed"));
}

store.close();
}

@Test
public void testSegmentsGenChecksums() throws IOException {
final ShardId shardId = new ShardId(new Index("index"), 1);
DirectoryService directoryService = new LuceneManagedDirectoryService(random(), false);
Store store = new Store(shardId, ImmutableSettings.EMPTY, null, directoryService, randomDistributor(directoryService));

// segments.gen: unspecified version
StoreFileMetaData segmentsGen = new StoreFileMetaData("segments.gen", 20, "boguschecksum", null);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp", segmentsGen, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
}

// segments.gen: specified old version
segmentsGen = new StoreFileMetaData("segments.gen", 20, "boguschecksum", Version.LUCENE_33);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp2", segmentsGen, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
}

// segments.gen: should still be checksummed for an ok version (lucene checksum)
segmentsGen = new StoreFileMetaData("segments.gen", 20, "boguschecksum", Version.LUCENE_48);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp3", segmentsGen, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
fail("should have gotten expected exception");
} catch (CorruptIndexException expected) {
assertTrue(expected.getMessage().startsWith("checksum failed"));
}

store.close();
}

@Test
public void testRenameFile() throws IOException {
final ShardId shardId = new ShardId(new Index("index"), 1);
Expand Down

0 comments on commit 0ad2a4a

Please sign in to comment.