Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Ignore the old checksum files (*checkSum.*). Read only from the .meta…

…data folder
  • Loading branch information...
commit 8378cec2459d3ed5be7ad987a8c7721105438cf8 1 parent d3867bf
@rsumbaly rsumbaly authored
View
13 contrib/hadoop-store-builder/src/java/voldemort/store/readonly/fetcher/HdfsFetcher.java
@@ -144,18 +144,7 @@ private boolean fetch(FileSystem fs, Path source, File dest, CopyStats stats)
// Kept for backwards compatibility
if(status.getPath().getName().contains("checkSum.txt")) {
- // Read checksum
- checkSumType = CheckSum.fromString(status.getPath().getName());
-
- // Get checksum generators
- checkSumGenerator = CheckSum.getInstance(checkSumType);
- fileCheckSumGenerator = CheckSum.getInstance(checkSumType);
-
- // Store original checksum to compare
- FSDataInputStream input = fs.open(status.getPath());
- origCheckSum = new byte[CheckSum.checkSumLength(checkSumType)];
- input.read(origCheckSum);
- input.close();
+ // Ignore old checksum files
} else if(status.getPath().getName().contains(".metadata")) {
View
2  contrib/hadoop-store-builder/test/voldemort/store/readonly/checksum/CheckSumTests.java
@@ -106,7 +106,7 @@ public int compare(File fs1, File fs2) {
for(File file: files) {
if(file.isFile() && !file.getName().startsWith(".")
- && !file.getName().contains("checkSum.txt")) {
+ && !file.getName().contains("checkSum.txt") && !file.getName().contains("metadata")) {
DataInputStream is;
try {
is = new DataInputStream(new FileInputStream(file));
View
48 contrib/hadoop-store-builder/test/voldemort/store/readonly/fetcher/HdfsFetcherTest.java
@@ -136,6 +136,8 @@ public void testCheckSumMetadata() throws Exception {
}
public void testFetch() throws Exception {
+ // Tests kept for backwards compatibility
+
File testSourceDirectory = TestUtils.createTempDir();
File testDestinationDirectory = TestUtils.createTempDir();
@@ -157,57 +159,13 @@ public void testFetch() throws Exception {
assertNotNull(fetchedFile);
checkSumFile.delete();
- // Test 3: Add checksum file with correct fileName, but empty = wrong
- // md5
+ // Test 3: Add checksum file with correct fileName, should be ignored
checkSumFile = new File(testSourceDirectory, "adler32checkSum.txt");
checkSumFile.createNewFile();
fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(),
testDestinationDirectory.getAbsolutePath() + "3");
- assertNull(fetchedFile);
-
- // Test 4: Add wrong contents to file i.e. contents of CRC32 instead of
- // Adler
- byte[] checkSumBytes = CheckSumTests.calculateCheckSum(testSourceDirectory.listFiles(),
- CheckSumType.CRC32);
- FileUtils.writeByteArrayToFile(checkSumFile, checkSumBytes);
- fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(),
- testDestinationDirectory.getAbsolutePath() + "4");
- assertNull(fetchedFile);
- checkSumFile.delete();
-
- // Test 5: Add correct checksum contents - MD5
- checkSumFile = new File(testSourceDirectory, "md5checkSum.txt");
- byte[] checkSumBytes2 = CheckSumTests.calculateCheckSum(testSourceDirectory.listFiles(),
- CheckSumType.MD5);
- FileUtils.writeByteArrayToFile(checkSumFile, checkSumBytes2);
- fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(),
- testDestinationDirectory.getAbsolutePath() + "5");
assertNotNull(fetchedFile);
checkSumFile.delete();
- // Test 6: Add correct checksum contents - ADLER32
- checkSumFile = new File(testSourceDirectory, "adler32checkSum.txt");
- byte[] checkSumBytes3 = CheckSumTests.calculateCheckSum(testSourceDirectory.listFiles(),
- CheckSumType.ADLER32);
- FileUtils.writeByteArrayToFile(checkSumFile, checkSumBytes3);
- fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(),
- testDestinationDirectory.getAbsolutePath() + "6");
- assertNotNull(fetchedFile);
- assertEquals(fetchedFile.getAbsolutePath(), testDestinationDirectory.getAbsolutePath()
- + "6");
- checkSumFile.delete();
-
- // Test 7: Add correct checksum contents - CRC32
- checkSumFile = new File(testSourceDirectory, "crc32checkSum.txt");
- byte[] checkSumBytes4 = CheckSumTests.calculateCheckSum(testSourceDirectory.listFiles(),
- CheckSumType.CRC32);
- FileUtils.writeByteArrayToFile(checkSumFile, checkSumBytes4);
- fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(),
- testDestinationDirectory.getAbsolutePath() + "7");
- assertNotNull(fetchedFile);
- assertEquals(fetchedFile.getAbsolutePath(), testDestinationDirectory.getAbsolutePath()
- + "7");
- checkSumFile.delete();
-
}
}
Please sign in to comment.
Something went wrong with that request. Please try again.