Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Core: Don't verify adler32 for lucene 3.x terms dict/terms index. #9142

Closed
wants to merge 6 commits into from
Closed

Core: Don't verify adler32 for lucene 3.x terms dict/terms index. #9142

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d138787
Don't verify adler32 for lucene 3.x terms dict/terms index.
rmuir Jan 5, 2015
c1c1027
Merge branch '1.x' into tis_tii_adler_fix
rmuir Jan 5, 2015
2fd1b02
enable 0.20.6 testing
rmuir Jan 5, 2015
895b64d
add paranoia
rmuir Jan 6, 2015
670cffa
add unit tests for different broken cases
rmuir Jan 6, 2015
5c4531d
test cleanups
rmuir Jan 6, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
23 changes: 20 additions & 3 deletions src/main/java/org/elasticsearch/index/store/Store.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -389,16 +389,33 @@ public static MetadataSnapshot readMetadataSnapshot(File[] indexLocations, ESLog
* and the metadata holds the necessary information to detect that it was been written by Lucene 4.8 or newer. If it has only
* a legacy checksum, returned IndexOutput will not verify the checksum.
* <p/>
* Note: Checksums are calculated nevertheless since lucene does it by default sicne version 4.8.0. This method only adds the
* Note: Checksums are calculated nevertheless since lucene does it by default since version 4.8.0. This method only adds the
* verification against the checksum in the given metadata and does not add any significant overhead.
*/
public IndexOutput createVerifyingOutput(String fileName, final StoreFileMetaData metadata, final IOContext context) throws IOException {
IndexOutput output = directory().createOutput(fileName, context);
boolean success = false;
try {
if (metadata.hasLegacyChecksum()) {
logger.debug("create legacy adler32 output for {}", fileName);
output = new LegacyVerification.Adler32VerifyingIndexOutput(output, metadata.checksum(), metadata.length());
// Lucene's .tii and .tis files (3.x) were not actually append-only.
// this means the adler32 in ES 0.20.x releases is actually wrong, we can't use it.
boolean badTermInfo = metadata.name().endsWith(".tii") || metadata.name().endsWith(".tis");
// Lucene's .cfs (3.0-3.3) was not append-only, so old ES checksums (pre-0.18.x) are wrong.
// NOTE: if we don't know the version, then we don't trust the checksum.
boolean badCFS = metadata.name().endsWith(".cfs") &&
(metadata.writtenBy() == null || metadata.writtenBy().onOrAfter(Version.LUCENE_34) == false);
// Lucene's segments_N always had a checksum, and reasonably old versions of ES never added
// their own metadata for it. But it wasn't append-only, so be defensive and don't rely upon
// old versions omitting the checksum. NOTE: This logic also excludes segments.gen for the same reason.
boolean badCommit = metadata.name().startsWith(IndexFileNames.SEGMENTS);

if (badTermInfo || badCFS || badCommit) {
logger.debug("create legacy length-only output for non-write-once file {}", fileName);
output = new LegacyVerification.LengthVerifyingIndexOutput(output, metadata.length());
} else {
logger.debug("create legacy adler32 output for {}", fileName);
output = new LegacyVerification.Adler32VerifyingIndexOutput(output, metadata.checksum(), metadata.length());
}
} else if (metadata.checksum() == null) {
// TODO: when the file is a segments_N, we can still CRC-32 + length for more safety
// its had that checksum forever.
Expand Down
3 changes: 1 addition & 2 deletions src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,7 @@
public class OldIndexBackwardsCompatibilityTests extends StaticIndexBackwardCompatibilityTest {

List<String> indexes = Arrays.asList(
// not yet: until https://github.com/elasticsearch/elasticsearch/pull/9142
// "index-0.20.6.zip",
"index-0.20.6.zip",
"index-0.90.0.zip",
"index-0.90.1.zip",
"index-0.90.2.zip",
Expand Down
146 changes: 146 additions & 0 deletions src/test/java/org/elasticsearch/index/store/StoreTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,152 @@ public void testMixedChecksums() throws IOException {
IOUtils.close(store);
}

// Test cases with incorrect adler32 in their metadata caused by old versions.

@Test
public void testBuggyTIIChecksums() throws IOException {
final ShardId shardId = new ShardId(new Index("index"), 1);
DirectoryService directoryService = new LuceneManagedDirectoryService(random(), false);
Store store = new Store(shardId, ImmutableSettings.EMPTY, null, directoryService, randomDistributor(directoryService), new DummyShardLock(shardId));

// .tii: no version specified
StoreFileMetaData tii = new StoreFileMetaData("foo.tii", 20, "boguschecksum", null);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp", tii, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
}

// .tii: old version
tii = new StoreFileMetaData("foo.tii", 20, "boguschecksum", Version.LUCENE_36);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp2", tii, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
}
store.close();
}

@Test
public void testBuggyTISChecksums() throws IOException {
final ShardId shardId = new ShardId(new Index("index"), 1);
DirectoryService directoryService = new LuceneManagedDirectoryService(random(), false);
Store store = new Store(shardId, ImmutableSettings.EMPTY, null, directoryService, randomDistributor(directoryService), new DummyShardLock(shardId));

// .tis: no version specified
StoreFileMetaData tis = new StoreFileMetaData("foo.tis", 20, "boguschecksum", null);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp", tis, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
};

// .tis: old version
tis = new StoreFileMetaData("foo.tis", 20, "boguschecksum", Version.LUCENE_36);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp", tis, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
};

store.close();
}

@Test
public void testBuggyCFSChecksums() throws IOException {
final ShardId shardId = new ShardId(new Index("index"), 1);
DirectoryService directoryService = new LuceneManagedDirectoryService(random(), false);
Store store = new Store(shardId, ImmutableSettings.EMPTY, null, directoryService, randomDistributor(directoryService), new DummyShardLock(shardId));

// .cfs: unspecified version
StoreFileMetaData cfs = new StoreFileMetaData("foo.cfs", 20, "boguschecksum", null);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp", cfs, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
}

// .cfs: ancient affected version
cfs = new StoreFileMetaData("foo.cfs", 20, "boguschecksum", Version.LUCENE_33);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp2", cfs, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
}

// .cfs: should still be checksummed for an ok version
cfs = new StoreFileMetaData("foo.cfs", 20, "boguschecksum", Version.LUCENE_34);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp3", cfs, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
fail("should have gotten expected exception");
} catch (CorruptIndexException expected) {
assertTrue(expected.getMessage().startsWith("checksum failed"));
}

store.close();
}

@Test
public void testSegmentsNChecksums() throws IOException {
final ShardId shardId = new ShardId(new Index("index"), 1);
DirectoryService directoryService = new LuceneManagedDirectoryService(random(), false);
Store store = new Store(shardId, ImmutableSettings.EMPTY, null, directoryService, randomDistributor(directoryService), new DummyShardLock(shardId));

// segments_N: unspecified version
StoreFileMetaData segments = new StoreFileMetaData("segments_1", 20, "boguschecksum", null);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp", segments, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
}

// segments_N: specified old version
segments = new StoreFileMetaData("segments_2", 20, "boguschecksum", Version.LUCENE_33);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp2", segments, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
}

// segments_N: should still be checksummed for an ok version (lucene checksum)
segments = new StoreFileMetaData("segments_3", 20, "boguschecksum", Version.LUCENE_48);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp3", segments, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
fail("should have gotten expected exception");
} catch (CorruptIndexException expected) {
assertTrue(expected.getMessage().startsWith("checksum failed"));
}

store.close();
}

@Test
public void testSegmentsGenChecksums() throws IOException {
final ShardId shardId = new ShardId(new Index("index"), 1);
DirectoryService directoryService = new LuceneManagedDirectoryService(random(), false);
Store store = new Store(shardId, ImmutableSettings.EMPTY, null, directoryService, randomDistributor(directoryService), new DummyShardLock(shardId));

// segments.gen: unspecified version
StoreFileMetaData segmentsGen = new StoreFileMetaData("segments.gen", 20, "boguschecksum", null);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp", segmentsGen, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
}

// segments.gen: specified old version
segmentsGen = new StoreFileMetaData("segments.gen", 20, "boguschecksum", Version.LUCENE_33);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp2", segmentsGen, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
}

// segments.gen: should still be checksummed for an ok version (lucene checksum)
segmentsGen = new StoreFileMetaData("segments.gen", 20, "boguschecksum", Version.LUCENE_48);
try (VerifyingIndexOutput output = (VerifyingIndexOutput) store.createVerifyingOutput("foo.temp3", segmentsGen, IOContext.DEFAULT)) {
output.writeBytes(new byte[20], 20);
output.verify();
fail("should have gotten expected exception");
} catch (CorruptIndexException expected) {
assertTrue(expected.getMessage().startsWith("checksum failed"));
}

store.close();
}

@Test
public void testRenameFile() throws IOException {
final ShardId shardId = new ShardId(new Index("index"), 1);
Expand Down