Skip to content
Browse files

Fix compatibility with lzop's checksums, and test same.

  • Loading branch information...
1 parent 0e670f4 commit 86fe2dbaeb9a8bf5e6f0071a0e0e4519d52f5f39 Shevek committed Aug 8, 2011
View
3 nbproject/project.properties
@@ -87,3 +87,6 @@ source.encoding=UTF-8
src.dir=src
src.jcpp.dir=jcpp
test.src.dir=test
+
+test-sys-prop.test.data.dir=${basedir}/test/data
+
View
9 src/org/anarres/lzo/LzopInputStream.java
@@ -96,7 +96,7 @@ public int getUncompressedChecksumCount() {
}
private void logState() {
- LOG.info("Flags = " + flags);
+ LOG.info("Flags = " + Integer.toHexString(flags));
LOG.info("CRC32C = " + c_crc32_c);
LOG.info("CRC32D = " + c_crc32_d);
LOG.info("Adler32C = " + c_adler32_c);
@@ -222,7 +222,6 @@ private int readChecksum(Checksum csum) throws IOException {
private void testChecksum(Checksum csum, int value, byte[] data, int off, int len) throws IOException {
if (csum == null)
return;
- // LOG.info("Testing checksum " + csum);
csum.reset();
csum.update(data, off, len);
if (value != (int) csum.getValue())
@@ -253,11 +252,11 @@ protected boolean readBlock() throws IOException {
int v_adler32_c = readChecksum(c_adler32_c);
int v_crc32_c = readChecksum(c_crc32_c);
readBytes(inputBuffer, 0, inputBufferLength);
- testChecksum(c_adler32_d, v_adler32_d, inputBuffer, 0, inputBufferLength);
- testChecksum(c_crc32_d, v_crc32_d, inputBuffer, 0, inputBufferLength);
- decompress(outputBufferLength, inputBufferLength);
testChecksum(c_adler32_c, v_adler32_c, inputBuffer, 0, inputBufferLength);
testChecksum(c_crc32_c, v_crc32_c, inputBuffer, 0, inputBufferLength);
+ decompress(outputBufferLength, inputBufferLength);
@shevek
shevek added a note Aug 9, 2011

DERP.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
+ testChecksum(c_adler32_d, v_adler32_d, outputBuffer, 0, outputBufferLength);
+ testChecksum(c_crc32_d, v_crc32_d, outputBuffer, 0, outputBufferLength);
return true;
}
}
View
12 src/org/apache/hadoop/io/compress/LzopCodec.java
@@ -0,0 +1,12 @@
+/*
+ * To change this template, choose Tools | Templates
@c0s
c0s added a note Aug 9, 2011

Perhaps you meant to change default template here ?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
+ * and open the template in the editor.
+ */
+package org.apache.hadoop.io.compress;
+
+/**
+ *
+ * @author shevek
+ */
+public class LzopCodec extends com.hadoop.compression.lzo.LzopCodec {
+}
View
5 test/data/compressed.txt
@@ -0,0 +1,5 @@
+foo a baz 1
+foo x baz 2
+foo y baz 3
+foo z baz 4
+foo b baz 5
View
BIN test/data/compressed.txt.lzo.adler32
Binary file not shown.
View
BIN test/data/compressed.txt.lzo.adler32d
Binary file not shown.
View
BIN test/data/compressed.txt.lzo.crc32
Binary file not shown.
View
BIN test/data/compressed.txt.lzo.crc32d
Binary file not shown.
View
57 test/org/anarres/lzo/LzopFileTest.java
@@ -0,0 +1,57 @@
+/*
+ * To change this template, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.anarres.lzo;
+
+import java.io.File;
+import java.io.FileInputStream;
+import org.apache.commons.io.IOUtils;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author shevek
+ */
+public class LzopFileTest {
+
+ private File getDataDirectory() {
+ String path = System.getProperty("test.data.dir");
+ File file = new File(path == null ? "test/data" : path);
+ assertTrue(file + " is a directory.", file.isDirectory());
+ return file;
+ }
+
+ private void testLzopFile(String name, long flags) throws Exception {
+ try {
+ File dir = getDataDirectory();
+ File lzo = new File(dir, name);
+ File raw = new File(dir, "compressed.txt");
+
+ FileInputStream lzo_fin = new FileInputStream(lzo);
+ LzopInputStream lzo_zin = new LzopInputStream(lzo_fin);
+ byte[] lzo_data = IOUtils.toByteArray(lzo_zin);
+
+ FileInputStream raw_fin = new FileInputStream(raw);
+ byte[] raw_data = IOUtils.toByteArray(raw_fin);
+
+ long tflags = lzo_zin.getFlags() & flags;
+ assertEquals("Missing flags in file", flags, tflags);
+
+ assertArrayEquals("Failed to decompress properly.", raw_data, lzo_data);
+ } finally {
+ System.out.flush();
+ System.err.flush();
+ Thread.sleep(100);
+ }
+ }
+
+ @Test
+ public void testLzopFile() throws Exception {
+ testLzopFile("compressed.txt.lzo.adler32d", LzopConstants.F_ADLER32_D);
+ testLzopFile("compressed.txt.lzo.crc32d", LzopConstants.F_CRC32_D);
+ testLzopFile("compressed.txt.lzo.adler32", LzopConstants.F_ADLER32_D | LzopConstants.F_ADLER32_C);
+ testLzopFile("compressed.txt.lzo.crc32", LzopConstants.F_CRC32_D | LzopConstants.F_CRC32_C);
+ }
+}

1 comment on commit 86fe2db

@c0s
c0s commented on 86fe2db Aug 9, 2011

I am glad you found that culprit! ;)

Please sign in to comment.
Something went wrong with that request. Please try again.