Permalink
Browse files

Added support for preset dictionaries at decompression time.

Closes #23
  • Loading branch information...
1 parent 9449d3f commit 8d1a4b659fe99424bdc38dac086e3f24f4307465 @jpountz committed Aug 30, 2013
View
4 CHANGES.md
@@ -4,6 +4,10 @@
- lz4 r102
+ - [#23](http://github.com/jpountz/lz4-java/issues/23)
+ Added the ability for the decompressors to use preset dictionaries as a 64kb
+ buffer in front of the buffer to decompress.
+
## 1.2.0
- lz4 r100
View
4 src/build/source_templates/decompress.template
@@ -1,5 +1,5 @@
@Override
- public int decompress(byte[] src, final int srcOff@if{ size == "Safe" }, final int srcLen @end{}, byte[] dest, final int destOff, int destLen) {
+ public int decompress@if{ prefix }WithPrefix64k@end{}(byte[] src, final int srcOff@if{ size == "Safe" }, final int srcLen @end{}, byte[] dest, final int destOff, int destLen) {
@if{ size == "Safe" }
checkRange(src, srcOff, srcLen);
checkRange(dest, destOff, destLen);
@@ -72,9 +72,11 @@
sOff += 2;
int matchOff = dOff - matchDec;
+@if{ !prefix }
if (matchOff < destOff) {
throw new LZ4Exception("Malformed input at " + sOff);
}
+@end{}
int matchLen = token & ML_MASK;
if (matchLen == ML_MASK) {
View
3 src/build/source_templates/decompressor.template
@@ -19,7 +19,8 @@ final class LZ4Java@{type}${size}Decompressor extends LZ4${size}Decompressor {
public static final LZ4${size}Decompressor INSTANCE = new LZ4Java${type}${size}Decompressor();
-@include{"decompress.template"}
+@include{"decompress.template"; prefix = false}
+@include{"decompress.template"; prefix = true}
}
View
9 src/java/net/jpountz/lz4/LZ4FastDecompressor.java
@@ -23,7 +23,7 @@
*/
public abstract class LZ4FastDecompressor implements LZ4Decompressor {
- /** Uncompress <code>src[srcOff:]</code> into <code>dest[destOff:destOff+destLen]</code>
+ /** Decompress <code>src[srcOff:]</code> into <code>dest[destOff:destOff+destLen]</code>
* and return the number of bytes read from <code>src</code>.
* <code>destLen</code> must be exactly the size of the decompressed data.
*
@@ -33,6 +33,13 @@
public abstract int decompress(byte[] src, int srcOff, byte[] dest, int destOff, int destLen);
/**
+ * Same as {@link #decompress(byte[], int, byte[], int, int)} except that up
+ * to 64 KB before <code>srcOff</code> in <code>src</code>. This is useful for
+ * providing LZ4 with a dictionary that can be reused during decompression.
+ */
+ public abstract int decompressWithPrefix64k(byte[] src, int srcOff, byte[] dest, int destOff, int destLen);
+
+ /**
* Convenience method, equivalent to calling
* {@link #decompress(byte[], int, byte[], int, int) decompress(src, 0, dest, 0, destLen)}.
*/
View
2 src/java/net/jpountz/lz4/LZ4JNI.java
@@ -32,7 +32,9 @@
static native int LZ4_compress_limitedOutput(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen);
static native int LZ4_compressHC(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen);
static native int LZ4_decompress_fast(byte[] src, int srcOff, byte[] dest, int destOff, int destLen);
+ static native int LZ4_decompress_fast_withPrefix64k(byte[] src, int srcOff, byte[] dest, int destOff, int destLen);
static native int LZ4_decompress_safe(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen);
+ static native int LZ4_decompress_safe_withPrefix64k(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen);
static native int LZ4_compressBound(int len);
}
View
10 src/java/net/jpountz/lz4/LZ4JNIFastDecompressor.java
@@ -35,4 +35,14 @@ public final int decompress(byte[] src, int srcOff, byte[] dest, int destOff, in
return result;
}
+ @Override
+ public final int decompressWithPrefix64k(byte[] src, int srcOff, byte[] dest, int destOff, int destLen) {
+ checkRange(src, srcOff);
+ checkRange(dest, destOff, destLen);
+ final int result = LZ4JNI.LZ4_decompress_fast_withPrefix64k(src, srcOff, dest, destOff, destLen);
+ if (result < 0) {
+ throw new LZ4Exception("Error decoding offset " + (srcOff - result) + " of input buffer");
+ }
+ return result;
+ }
}
View
11 src/java/net/jpountz/lz4/LZ4JNISafeDecompressor.java
@@ -34,4 +34,15 @@ public final int decompress(byte[] src, int srcOff, int srcLen, byte[] dest, int
}
return result;
}
+
+ @Override
+ public final int decompressWithPrefix64k(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen) {
+ checkRange(src, srcOff, srcLen);
+ checkRange(dest, destOff, maxDestLen);
+ final int result = LZ4JNI.LZ4_decompress_safe_withPrefix64k(src, srcOff, srcLen, dest, destOff, maxDestLen);
+ if (result < 0) {
+ throw new LZ4Exception("Error decoding offset " + (srcOff - result) + " of input buffer");
+ }
+ return result;
+ }
}
View
15 src/java/net/jpountz/lz4/LZ4SafeDecompressor.java
@@ -37,6 +37,13 @@
public abstract int decompress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen);
/**
+ * Same as {@link #decompress(byte[], int, int, byte[], int, int) except that
+ * up to 64 KB before <code>srcOff</code> in <code>src</code>. This is useful
+ * for providing LZ4 with a dictionary that can be reused during decompression.
+ */
+ public abstract int decompressWithPrefix64k(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen);
+
+ /**
* Convenience method, equivalent to calling
* {@link #decompress(byte[], int, int, byte[], int, int) decompress(src, srcOff, srcLen, dest, destOff, dest.length - destOff)}.
*/
@@ -46,6 +53,14 @@ public final int decompress(byte[] src, int srcOff, int srcLen, byte[] dest, int
/**
* Convenience method, equivalent to calling
+ * {@link #decompressWithPrefix64k(byte[], int, int, byte[], int, int) decompress(src, srcOff, srcLen, dest, destOff, dest.length - destOff)}.
+ */
+ public final int decompressWithPrefix64k(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff) {
+ return decompressWithPrefix64k(src, srcOff, srcLen, dest, destOff, dest.length - destOff);
+ }
+
+ /**
+ * Convenience method, equivalent to calling
* {@link #decompress(byte[], int, int, byte[], int) decompress(src, 0, src.length, dest, 0)}
*/
public final int decompress(byte[] src, byte[] dest) {
View
56 src/jni/net_jpountz_lz4_LZ4JNI.c
@@ -145,6 +145,62 @@ JNIEXPORT jint JNICALL Java_net_jpountz_lz4_LZ4JNI_LZ4_1decompress_1safe
/*
* Class: net_jpountz_lz4_LZ4JNI
+ * Method: LZ4_decompress_fast_withPrefix64k
+ * Signature: ([BI[BII)I
+ */
+JNIEXPORT jint JNICALL Java_net_jpountz_lz4_LZ4JNI_LZ4_1decompress_1fast_1withPrefix64k
+ (JNIEnv *env, jclass cls, jbyteArray src, jint srcOff, jbyteArray dest, jint destOff, jint destLen) {
+
+ char* in = (char*) (*env)->GetPrimitiveArrayCritical(env, src, 0);
+ if (in == NULL) {
+ throw_OOM(env);
+ return 0;
+ }
+ char* out = (char*) (*env)->GetPrimitiveArrayCritical(env, dest, 0);
+ if (out == NULL) {
+ throw_OOM(env);
+ return 0;
+ }
+
+ jint compressed = LZ4_decompress_fast_withPrefix64k(in + srcOff, out + destOff, destLen);
+
+ (*env)->ReleasePrimitiveArrayCritical(env, src, in, 0);
+ (*env)->ReleasePrimitiveArrayCritical(env, src, out, 0);
+
+ return compressed;
+
+}
+
+/*
+ * Class: net_jpountz_lz4_LZ4JNI
+ * Method: LZ4_decompress_safe_withPrefix64k
+ * Signature: ([BII[BII)I
+ */
+JNIEXPORT jint JNICALL Java_net_jpountz_lz4_LZ4JNI_LZ4_1decompress_1safe_1withPrefix64k
+ (JNIEnv *env, jclass cls, jbyteArray src, jint srcOff, jint srcLen, jbyteArray dest, jint destOff, jint maxDestLen) {
+
+ char* in = (char*) (*env)->GetPrimitiveArrayCritical(env, src, 0);
+ if (in == NULL) {
+ throw_OOM(env);
+ return 0;
+ }
+ char* out = (char*) (*env)->GetPrimitiveArrayCritical(env, dest, 0);
+ if (out == NULL) {
+ throw_OOM(env);
+ return 0;
+ }
+
+ jint decompressed = LZ4_decompress_safe_withPrefix64k(in + srcOff, out + destOff, srcLen, maxDestLen);
+
+ (*env)->ReleasePrimitiveArrayCritical(env, src, in, 0);
+ (*env)->ReleasePrimitiveArrayCritical(env, src, out, 0);
+
+ return decompressed;
+
+}
+
+/*
+ * Class: net_jpountz_lz4_LZ4JNI
* Method: LZ4_compressBound
* Signature: (I)I
*/
View
33 src/test/net/jpountz/lz4/LZ4Test.java
@@ -130,6 +130,11 @@ public void testRoundTrip(byte[] data, int off, int len,
assertEquals(compressedLen, decompressor.decompress(compressed, 0, restored, 0, len));
assertArrayEquals(Arrays.copyOfRange(data, off, off + len), restored);
+ // test decompression with prefix
+ Arrays.fill(restored, (byte) randomByte());
+ assertEquals(compressedLen, decompressor.decompressWithPrefix64k(compressed, 0, restored, 0, len));
+ assertArrayEquals(Arrays.copyOfRange(data, off, off + len), restored);
+
if (len > 0) {
// dest is too small
try {
@@ -151,11 +156,14 @@ public void testRoundTrip(byte[] data, int off, int len,
// try decompression when only the size of the compressed buffer is known
if (len > 0) {
- Arrays.fill(restored, (byte) 0);
- decompressor2.decompress(compressed, 0, compressedLen, restored, 0);
+ Arrays.fill(restored, randomByte());
assertEquals(len, decompressor2.decompress(compressed, 0, compressedLen, restored, 0));
+
+ Arrays.fill(restored, randomByte());
+ assertEquals(len, decompressor2.decompressWithPrefix64k(compressed, 0, compressedLen, restored, 0));
} else {
assertEquals(0, decompressor2.decompress(compressed, 0, compressedLen, new byte[1], 0));
+ assertEquals(0, decompressor2.decompressWithPrefix64k(compressed, 0, compressedLen, new byte[1], 0));
}
// over-estimated compressed length
@@ -391,6 +399,27 @@ public void testRoundtripIssue12() {
testRoundTrip(data, 9, data.length - 9);
}
+ @Test
+ public void testDecompressWithPrefix64k() {
+ final byte[] compressed = new byte[] {
+ 16, 42, 7,0, 80, 1,2,3,4,5
+ };
+ final byte[] original = new byte[] {
+ 42,1,2,3,4,1,2,3,4,5
+ };
+ for (LZ4FastDecompressor decompressor : FAST_DECOMPRESSORS) {
+ final byte[] restored = new byte[16];
+ restored[0] = 1;
+ restored[1] = 2;
+ restored[2] = 3;
+ restored[3] = 4;
+ restored[4] = 5;
+ final int compressedLen = decompressor.decompressWithPrefix64k(compressed, 0, restored, 6, restored.length - 6);
+ assertEquals(compressed.length, compressedLen);
+ assertArrayEquals(original, Arrays.copyOfRange(restored, 6, restored.length));
+ }
+ }
+
private static void assertCompressedArrayEquals(String message, byte[] expected, byte[] actual) {
int off = 0;
int decompressedOff = 0;

0 comments on commit 8d1a4b6

Please sign in to comment.