Skip to content

Commit

Permalink
Merge branch 'master' of github.com:linnaea/lz4-java into linnaea-master
Browse files Browse the repository at this point in the history 
Conflicts:
	src/java/net/jpountz/lz4/LZ4HCJNICompressor.java
  • Loading branch information
@jpountz
jpountz committed Oct 12, 2014
2 parents 811356a + a619374 commit 43408f9
Show file tree
Hide file tree
Showing 8 changed files with 80 additions and 14 deletions.
16 changes: 12 additions & 4 deletions src/build/source_templates/compressor_hc.template
View file
Expand Up @@ -29,8 +29,16 @@ final class LZ4HCJava${type}@{TypeSuffix}Compressor extends LZ4Compressor {

public static final LZ4Compressor INSTANCE = new LZ4HCJava${type}@{TypeSuffix}Compressor();

static class HashTable {
static final int MAX_ATTEMPTS = 256;
private final int maxAttempts;
final int compressionLevel;

LZ4HCJava${type}@{TypeSuffix}Compressor() { this(DEFAULT_COMPRESSION_LEVEL); }
LZ4HCJava${type}@{TypeSuffix}Compressor(int compressionLevel) {
this.maxAttempts = 1<<(compressionLevel-1);
this.compressionLevel = compressionLevel;
}

private class HashTable {
static final int MASK = MAX_DISTANCE - 1;
@{OffsetType} nextToUpdate;
private final @{OffsetType} base;
Expand Down Expand Up @@ -92,7 +100,7 @@ final class LZ4HCJava${type}@{TypeSuffix}Compressor extends LZ4Compressor {
ref = next(ref);
}

for (int i = 0; i < MAX_ATTEMPTS; ++i) {
for (int i = 0; i < maxAttempts; ++i) {
if (ref < Math.max(base, off - MAX_DISTANCE + 1) || ref > off) {
break;
}
Expand Down Expand Up @@ -131,7 +139,7 @@ final class LZ4HCJava${type}@{TypeSuffix}Compressor extends LZ4Compressor {

final @{OffsetType} delta = off - startLimit;
@{OffsetType} ref = hashPointer(buf, off);
for (int i = 0; i < MAX_ATTEMPTS; ++i) {
for (int i = 0; i < maxAttempts; ++i) {
if (ref < Math.max(base, off - MAX_DISTANCE + 1) || ref > off) {
break;
}
Expand Down
3 changes: 3 additions & 0 deletions src/java/net/jpountz/lz4/LZ4Constants.java
View file
Expand Up @@ -18,6 +18,9 @@
enum LZ4Constants {
;

static final int DEFAULT_COMPRESSION_LEVEL = 8+1;
static final int MAX_COMPRESSION_LEVEL = 16+1;

static final int MEMORY_USAGE = 14;
static final int NOT_COMPRESSIBLE_DETECTION_LEVEL = 6;

Expand Down
31 changes: 30 additions & 1 deletion src/java/net/jpountz/lz4/LZ4Factory.java
View file
Expand Up @@ -14,12 +14,16 @@
* limitations under the License.
*/

import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.util.Arrays;

import net.jpountz.util.Native;
import net.jpountz.util.UnsafeBase;
import net.jpountz.util.Utils;
import static net.jpountz.lz4.LZ4Constants.DEFAULT_COMPRESSION_LEVEL;
import static net.jpountz.lz4.LZ4Constants.MAX_COMPRESSION_LEVEL;

/**
* Entry point for the LZ4 API.
Expand Down Expand Up @@ -153,13 +157,20 @@ private static <T> T classInstance(String cls) throws NoSuchFieldException, Secu
private final LZ4Compressor highCompressor;
private final LZ4FastDecompressor fastDecompressor;
private final LZ4SafeDecompressor safeDecompressor;
private final LZ4Compressor[] highCompressors = new LZ4Compressor[MAX_COMPRESSION_LEVEL+1];

private LZ4Factory(String impl) throws ClassNotFoundException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException {
private LZ4Factory(String impl) throws ClassNotFoundException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException, NoSuchMethodException, InstantiationException, InvocationTargetException {
this.impl = impl;
fastCompressor = classInstance("net.jpountz.lz4.LZ4" + impl + "Compressor");
highCompressor = classInstance("net.jpountz.lz4.LZ4HC" + impl + "Compressor");
fastDecompressor = classInstance("net.jpountz.lz4.LZ4" + impl + "FastDecompressor");
safeDecompressor = classInstance("net.jpountz.lz4.LZ4" + impl + "SafeDecompressor");
Constructor<? extends LZ4Compressor> highConstructor = highCompressor.getClass().getDeclaredConstructor(int.class);
highCompressors[DEFAULT_COMPRESSION_LEVEL] = highCompressor;
for(int level = 1; level <= MAX_COMPRESSION_LEVEL; level++) {
if(level == DEFAULT_COMPRESSION_LEVEL) continue;
highCompressors[level] = highConstructor.newInstance(level);
}

// quickly test that everything works as expected
final byte[] original = new byte[] {'a','b','c','d',' ',' ',' ',' ',' ',' ','a','b','c','d','e','f','g','h','i','j'};
Expand Down Expand Up @@ -192,6 +203,24 @@ public LZ4Compressor highCompressor() {
return highCompressor;
}

/** Return a {@link LZ4Compressor} which requires more memory than
* {@link #fastCompressor()} and is slower but compresses more efficiently.
* The compression level can be customized.
* <p>For current implementations, the following is true about compression level:<ol>
* <li>It should be in range [1, 17]</li>
* <li>A compression level higher than 17 would be treated as 17.</li>
* <li>A compression level lower than 1 would be treated as 9.</li>
* </ol></p>
*/
public LZ4Compressor highCompressor(int compressionLevel) {
if(compressionLevel > MAX_COMPRESSION_LEVEL) {
compressionLevel = MAX_COMPRESSION_LEVEL;
} else if(compressionLevel < 1) {
compressionLevel = DEFAULT_COMPRESSION_LEVEL;
}
return highCompressors[compressionLevel];
}

/** Return a {@link LZ4FastDecompressor} instance. */
public LZ4FastDecompressor fastDecompressor() {
return fastDecompressor;
Expand Down
14 changes: 11 additions & 3 deletions src/java/net/jpountz/lz4/LZ4HCJNICompressor.java
View file
Expand Up @@ -16,6 +16,7 @@

import static net.jpountz.util.ByteBufferUtils.checkNotReadOnly;
import static net.jpountz.util.ByteBufferUtils.checkRange;
import static net.jpountz.lz4.LZ4Constants.DEFAULT_COMPRESSION_LEVEL;
import static net.jpountz.util.Utils.checkRange;

import java.nio.ByteBuffer;
Expand All @@ -30,11 +31,18 @@ final class LZ4HCJNICompressor extends LZ4Compressor {

public static final LZ4Compressor INSTANCE = new LZ4HCJNICompressor();

private final int compressionLevel;

LZ4HCJNICompressor() { this(DEFAULT_COMPRESSION_LEVEL); }
LZ4HCJNICompressor(int compressionLevel) {
this.compressionLevel = compressionLevel;
}

@Override
public int compress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen) {
checkRange(src, srcOff, srcLen);
checkRange(dest, destOff, maxDestLen);
final int result = LZ4JNI.LZ4_compressHC(src, null, srcOff, srcLen, dest, null, destOff, maxDestLen);
final int result = LZ4JNI.LZ4_compressHC(src, null, srcOff, srcLen, dest, null, destOff, maxDestLen, compressionLevel);
if (result <= 0) {
throw new LZ4Exception();
}
Expand All @@ -48,13 +56,13 @@ public int compress(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int
checkNotReadOnly(dest);
if (!src.isDirect() && src.isReadOnly()) {
// JNI can't access data in this case. Fall back to Java implementation.
return LZ4Factory.fastestJavaInstance().highCompressor().
return LZ4Factory.safeInstance().highCompressor(compressionLevel).
compress(src, srcOff, srcLen, dest, destOff, maxDestLen);
}

int result = LZ4JNI.LZ4_compressHC(
ByteBufferUtils.getArray(src), src, srcOff, srcLen,
ByteBufferUtils.getArray(dest), dest, destOff, maxDestLen);
ByteBufferUtils.getArray(dest), dest, destOff, maxDestLen, compressionLevel);
if (result <= 0) {
throw new LZ4Exception();
}
Expand Down
2 changes: 1 addition & 1 deletion src/java/net/jpountz/lz4/LZ4JNI.java
View file
Expand Up @@ -32,7 +32,7 @@ enum LZ4JNI {

static native void init();
static native int LZ4_compress_limitedOutput(byte[] srcArray, ByteBuffer srcBuffer, int srcOff, int srcLen, byte[] destArray, ByteBuffer destBuffer, int destOff, int maxDestLen);
static native int LZ4_compressHC(byte[] srcArray, ByteBuffer srcBuffer, int srcOff, int srcLen, byte[] destArray, ByteBuffer destBuffer, int destOff, int maxDestLen);
static native int LZ4_compressHC(byte[] srcArray, ByteBuffer srcBuffer, int srcOff, int srcLen, byte[] destArray, ByteBuffer destBuffer, int destOff, int maxDestLen, int compressionLevel);
static native int LZ4_decompress_fast(byte[] srcArray, ByteBuffer srcBuffer, int srcOff, byte[] destArray, ByteBuffer destBuffer, int destOff, int destLen);
static native int LZ4_decompress_fast_withPrefix64k(byte[] srcArray, ByteBuffer srcBuffer, int srcOff, byte[] destArray, ByteBuffer destBuffer, int destOff, int destLen);
static native int LZ4_decompress_safe(byte[] srcArray, ByteBuffer srcBuffer, int srcOff, int srcLen, byte[] destArray, ByteBuffer destBuffer, int destOff, int maxDestLen);
Expand Down
5 changes: 3 additions & 2 deletions src/jni/net_jpountz_lz4_LZ4JNI.c
View file
Expand Up @@ -13,6 +13,7 @@
*/

#include "lz4.h"
#include "lz4hc.h"
#include "net_jpountz_lz4_LZ4JNI.h"

static jclass OutOfMemoryError;
Expand Down Expand Up @@ -81,7 +82,7 @@ JNIEXPORT jint JNICALL Java_net_jpountz_lz4_LZ4JNI_LZ4_1compress_1limitedOutput
* Signature: ([BII[BI)I
*/
JNIEXPORT jint JNICALL Java_net_jpountz_lz4_LZ4JNI_LZ4_1compressHC
(JNIEnv *env, jclass cls, jbyteArray srcArray, jobject srcBuffer, jint srcOff, jint srcLen, jbyteArray destArray, jobject destBuffer, jint destOff, jint maxDestLen) {
(JNIEnv *env, jclass cls, jbyteArray srcArray, jobject srcBuffer, jint srcOff, jint srcLen, jbyteArray destArray, jobject destBuffer, jint destOff, jint maxDestLen, jint compressionLevel) {

char* in;
char* out;
Expand All @@ -106,7 +107,7 @@ JNIEXPORT jint JNICALL Java_net_jpountz_lz4_LZ4JNI_LZ4_1compressHC
return 0;
}

compressed = LZ4_compressHC_limitedOutput(in + srcOff, out + destOff, srcLen, maxDestLen);
compressed = LZ4_compressHC2_limitedOutput(in + srcOff, out + destOff, srcLen, maxDestLen, compressionLevel);

if (destArray != NULL) {
(*env)->ReleasePrimitiveArrayCritical(env, destArray, out, 0);
Expand Down
10 changes: 7 additions & 3 deletions src/test/net/jpountz/lz4/AbstractLZ4RoundtripTest.java
View file
Expand Up @@ -89,6 +89,10 @@ LZ4Compressor refCompressor() {
} else if (compressor == LZ4Factory.unsafeInstance().highCompressor()
|| compressor == LZ4Factory.safeInstance().highCompressor()) {
return LZ4Factory.nativeInstance().highCompressor();
} else if (compressor instanceof LZ4HCJavaSafeCompressor) {
return LZ4Factory.nativeInstance().highCompressor(((LZ4HCJavaSafeCompressor)compressor).compressionLevel);
} else if (compressor instanceof LZ4HCJavaUnsafeCompressor) {
return LZ4Factory.nativeInstance().highCompressor(((LZ4HCJavaUnsafeCompressor)compressor).compressionLevel);
}
return null;
}
Expand Down Expand Up @@ -333,9 +337,9 @@ public void testRoundTrip(byte[] data, int off, int len,

public void testRoundTrip(byte[] data, int off, int len,
LZ4Factory lz4) {
for (LZ4Compressor compressor : Arrays.asList(
lz4.fastCompressor(), lz4.highCompressor())) {
testRoundTrip(data, off, len, compressor, lz4.fastDecompressor(), lz4.safeDecompressor());
testRoundTrip(data, off, len, lz4.fastCompressor(), lz4.fastDecompressor(), lz4.safeDecompressor());
for (int level : Arrays.asList(1, 5, 9, 13)) { //Test compression level 1, 5, 9(default) and 13 only. Should be ok.
testRoundTrip(data, off, len, lz4.highCompressor(level), lz4.fastDecompressor(), lz4.safeDecompressor());
}
}

Expand Down
13 changes: 13 additions & 0 deletions src/test/net/jpountz/lz4/LZ4FactoryTest.java
View file
Expand Up @@ -31,6 +31,19 @@ public void test() {
assertEquals(LZ4JNISafeDecompressor.INSTANCE, LZ4Factory.nativeInstance().safeDecompressor());
assertEquals(LZ4JavaSafeSafeDecompressor.INSTANCE, LZ4Factory.safeInstance().safeDecompressor());

//Calling with default level should return the same one as the constructor without a level option.
assertEquals(LZ4Factory.nativeInstance().highCompressor(LZ4Constants.DEFAULT_COMPRESSION_LEVEL), LZ4Factory.nativeInstance().highCompressor());
assertEquals(LZ4Factory.unsafeInstance().highCompressor(LZ4Constants.DEFAULT_COMPRESSION_LEVEL), LZ4Factory.unsafeInstance().highCompressor());
assertEquals(LZ4Factory.safeInstance().highCompressor(LZ4Constants.DEFAULT_COMPRESSION_LEVEL), LZ4Factory.safeInstance().highCompressor());
//Calling with a level too high should give back the highest possible one.
assertEquals(LZ4Factory.nativeInstance().highCompressor(LZ4Constants.MAX_COMPRESSION_LEVEL+1), LZ4Factory.nativeInstance().highCompressor(LZ4Constants.MAX_COMPRESSION_LEVEL));
assertEquals(LZ4Factory.unsafeInstance().highCompressor(LZ4Constants.MAX_COMPRESSION_LEVEL+1), LZ4Factory.unsafeInstance().highCompressor(LZ4Constants.MAX_COMPRESSION_LEVEL));
assertEquals(LZ4Factory.safeInstance().highCompressor(LZ4Constants.MAX_COMPRESSION_LEVEL+1), LZ4Factory.safeInstance().highCompressor(LZ4Constants.MAX_COMPRESSION_LEVEL));
//Calling with a level less than 1 should give back the default one.
assertEquals(LZ4Factory.nativeInstance().highCompressor(LZ4Constants.DEFAULT_COMPRESSION_LEVEL), LZ4Factory.nativeInstance().highCompressor(0));
assertEquals(LZ4Factory.unsafeInstance().highCompressor(LZ4Constants.DEFAULT_COMPRESSION_LEVEL), LZ4Factory.unsafeInstance().highCompressor(0));
assertEquals(LZ4Factory.safeInstance().highCompressor(LZ4Constants.DEFAULT_COMPRESSION_LEVEL), LZ4Factory.safeInstance().highCompressor(0));

if ("Long".equals(UnsafeBase.POINTER_SIZE_SUFFIX)) {
assertEquals(LZ4JavaUnsafeLongCompressor.INSTANCE, LZ4Factory.unsafeInstance().fastCompressor());
assertEquals(LZ4HCJavaUnsafeLongCompressor.INSTANCE, LZ4Factory.unsafeInstance().highCompressor());
Expand Down

0 comments on commit 43408f9

Please sign in to comment.