Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

initial stream support added

  • Loading branch information...
commit 909dd70164a53aaeef7a2faf71fce596edf7286f 1 parent 936755c
Jon Hartlaub authored
View
18 .gitignore
@@ -0,0 +1,18 @@
+*.iml
+*.ipr
+*.iws
+*.log
+.DS_Store
+.classpath
+.settings
+.project
+target
+pom.xml.releaseBackup
+release.properties
+*~
+temp-testng-customsuite.xml
+test-output
+.externalToolBuilders
+server/logs
+runtime
+logs
View
7 pom.xml
@@ -51,6 +51,13 @@ See "http://oldhome.schmorp.de/marc/liblzf.html" for more on original LZF packag
</license>
</licenses>
<dependencies>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <version>5.12.1</version>
+ <type>jar</type>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<defaultGoal>install</defaultGoal>
View
66 src/main/java/com/ning/compress/lzf/LZFDecoder.java
@@ -12,6 +12,7 @@
package com.ning.compress.lzf;
import java.io.IOException;
+import java.io.InputStream;
/**
* Decoder that handles decoding of sequence of encoded LZF chunks,
@@ -21,44 +22,52 @@
*/
public class LZFDecoder
{
- private final static byte BYTE_NULL = 0;
+ private final static byte BYTE_NULL = 0;
+ private final static int HEADER_BYTES = 5;
// static methods, no need to instantiate
private LZFDecoder() { }
+ public static byte[] decode(final byte[] sourceBuffer) throws IOException
+ {
+ byte[] result = new byte[calculateUncompressedSize(sourceBuffer)];
+ decode(sourceBuffer, result);
+ return result;
+ }
+
/**
* Method for decompressing whole input data, which encoded in LZF
* block structure (compatible with lzf command line utility),
* and can consist of any number of blocks
*/
- public static byte[] decode(byte[] data) throws IOException
+ public static int decode(final byte[] sourceBuffer, final byte[] targetBuffer) throws IOException
{
/* First: let's calculate actual size, so we can allocate
* exact result size. Also useful for basic sanity checking;
* so that after call we know header structure is not corrupt
* (to the degree that lengths etc seem valid)
*/
- byte[] result = new byte[calculateUncompressedSize(data)];
+ byte[] result = targetBuffer;
int inPtr = 0;
int outPtr = 0;
- while (inPtr < (data.length - 1)) { // -1 to offset possible end marker
+ while (inPtr < (sourceBuffer.length - 1)) { // -1 to offset possible end marker
inPtr += 2; // skip 'ZV' marker
- int type = data[inPtr++];
- int len = uint16(data, inPtr);
+ int type = sourceBuffer[inPtr++];
+ int len = uint16(sourceBuffer, inPtr);
inPtr += 2;
if (type == LZFChunk.BLOCK_TYPE_NON_COMPRESSED) { // uncompressed
- System.arraycopy(data, inPtr, result, outPtr, len);
+ System.arraycopy(sourceBuffer, inPtr, result, outPtr, len);
outPtr += len;
} else { // compressed
- int uncompLen = uint16(data, inPtr);
+ int uncompLen = uint16(sourceBuffer, inPtr);
inPtr += 2;
- decompressChunk(data, inPtr, result, outPtr, outPtr+uncompLen);
+ decompressChunk(sourceBuffer, inPtr, result, outPtr, outPtr+uncompLen);
outPtr += uncompLen;
}
inPtr += len;
}
- return result;
+ return outPtr;
}
private static int calculateUncompressedSize(byte[] data) throws IOException
@@ -103,6 +112,43 @@ private static int calculateUncompressedSize(byte[] data) throws IOException
}
/**
+ * Main decode from a stream. Decompressed bytes are placed in the outputBuffer, inputBuffer is a "scratch-area".
+ *
+ * @param is An input stream of LZF compressed bytes
+ * @param inputBuffer A byte array used as a scratch area.
+ * @param outputBuffer A byte array in which the result is returned
+ * @return The number of bytes placed in the outputBuffer.
+ */
+ public static int decompressChunk(final InputStream is, final byte[] inputBuffer, final byte[] outputBuffer)
+ throws IOException
+ {
+ int bytesInOutput;
+ int headerLength = is.read(inputBuffer, 0, HEADER_BYTES);
+ if(headerLength != HEADER_BYTES) {
+ return -1;
+ }
+ int inPtr =0;
+ if (inputBuffer[inPtr] != LZFChunk.BYTE_Z || inputBuffer[inPtr+1] != LZFChunk.BYTE_V) {
+ throw new IOException("Corrupt input data, block did not start with 'ZV' signature bytes");
+ }
+ inPtr += 2;
+ int type = inputBuffer[inPtr++];
+ int len = uint16(inputBuffer, inPtr);
+ inPtr += 2;
+ if (type == LZFChunk.BLOCK_TYPE_NON_COMPRESSED) { // uncompressed
+ is.read(outputBuffer, 0, len);
+ bytesInOutput = len;
+ } else { // compressed
+ is.read(inputBuffer, inPtr, 2);
+ int uncompLen = uint16(inputBuffer, inPtr);
+ is.read(inputBuffer, 0, len);
+ decompressChunk(inputBuffer, 0, outputBuffer, 0, uncompLen);
+ bytesInOutput = uncompLen;
+ }
+ return bytesInOutput;
+ }
+
+ /**
* Main decode method for individual chunks.
*/
public static void decompressChunk(byte[] in, int inPos, byte[] out, int outPos, int outEnd)
View
9 src/main/java/com/ning/compress/lzf/LZFEncoder.java
@@ -25,14 +25,19 @@
// Static methods only, no point in instantiating
private LZFEncoder() { }
+ public static byte[] encode(byte[] data) throws IOException
+ {
+ return encode(data, data.length);
+ }
+
/**
* Method for compressing given input data using LZF encoding and
* block structure (compatible with lzf command line utility).
* Result consists of a sequence of chunks.
*/
- public static byte[] encode(byte[] data) throws IOException
+ public static byte[] encode(byte[] data, int length) throws IOException
{
- int left = data.length;
+ int left = length;
ChunkEncoder enc = new ChunkEncoder(left);
int chunkLen = Math.min(LZFChunk.MAX_CHUNK_LEN, left);
LZFChunk first = enc.encodeChunk(data, 0, chunkLen);
View
83 src/main/java/com/ning/compress/lzf/LZFInputStream.java
@@ -0,0 +1,83 @@
+package com.ning.compress.lzf;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class LZFInputStream extends InputStream
+{
+ public static int EOF_FLAG = -1;
+
+ /* stream to be decompressed */
+ private final InputStream inputStream;
+
+ /* the current buffer of compressed bytes */
+ private final byte[] compressedBytes = new byte[LZFChunk.MAX_CHUNK_LEN];
+
+ /* the buffer of uncompressed bytes from which */
+ private final byte[] uncompressedBytes = new byte[LZFChunk.MAX_CHUNK_LEN];
+
+ /* The current position (next char to output) in the uncompressed bytes buffer. */
+ private int bufferPosition = 0;
+
+ /* Length of the current uncompressed bytes buffer */
+ private int bufferLength = 0;
+
+ public LZFInputStream(final InputStream inputStream) throws IOException
+ {
+ super();
+ this.inputStream = inputStream;
+ }
+
+ @Override
+ public int read() throws IOException {
+ int returnValue = EOF_FLAG;
+ readyBuffer();
+ if(bufferPosition < bufferLength) {
+ returnValue = (uncompressedBytes[bufferPosition++] & 255);
+ }
+ return returnValue;
+ }
+
+ public int read(final byte[] buffer) throws IOException
+ {
+ return(read(buffer, 0, buffer.length));
+
+ }
+
+ public int read(final byte[] buffer, final int offset, final int length) throws IOException
+ {
+ int outputPos = offset;
+ readyBuffer();
+ if(bufferLength == -1)
+ {
+ return -1;
+ }
+
+ while(outputPos < buffer.length && bufferPosition < bufferLength) {
+ int chunkLength = Math.min(bufferLength - bufferPosition, buffer.length - outputPos);
+ System.arraycopy(uncompressedBytes, bufferPosition, buffer, outputPos, chunkLength);
+ outputPos += chunkLength;
+ bufferPosition += chunkLength;
+ readyBuffer();
+ }
+ return outputPos;
+ }
+
+ public void close() throws IOException
+ {
+ inputStream.close();
+ }
+
+ /**
+ * Fill the uncompressed bytes buffer by reading the underlying inputStream.
+ * @throws IOException
+ */
+ private void readyBuffer() throws IOException
+ {
+ if(bufferPosition >= bufferLength)
+ {
+ bufferLength = LZFDecoder.decompressChunk(inputStream, compressedBytes, uncompressedBytes);
+ bufferPosition = 0;
+ }
+ }
+}
View
76 src/main/java/com/ning/compress/lzf/LZFOutputStream.java
@@ -0,0 +1,76 @@
+package com.ning.compress.lzf;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class LZFOutputStream extends OutputStream
+{
+ private static int OUTPUT_BUFFER_SIZE = LZFChunk.MAX_CHUNK_LEN;
+ private static int BYTE_MASK = 0xff;
+
+ private final OutputStream outputStream;
+
+ private byte[] outputBuffer = new byte[OUTPUT_BUFFER_SIZE];
+ private int position = 0;
+
+ public LZFOutputStream(final OutputStream outputStream)
+ {
+ this.outputStream = outputStream;
+ }
+
+ @Override
+ public void write(final int singleByte) throws IOException
+ {
+ if(position >= outputBuffer.length) {
+ writeCompressedBlock();
+ }
+ outputBuffer[position++] = (byte) (singleByte & BYTE_MASK);
+ }
+
+ @Override
+ public void write(final byte[] buffer, final int offset, final int length) throws IOException
+ {
+ int inputCursor = offset;
+ int remainingBytes = length;
+ while(remainingBytes > 0) {
+ if(position >= outputBuffer.length) {
+ writeCompressedBlock();
+ }
+ int chunkLength = (remainingBytes > (outputBuffer.length - position))?outputBuffer.length - position: remainingBytes;
+ System.arraycopy(buffer, inputCursor, outputBuffer, position, chunkLength);
+ position += chunkLength;
+ remainingBytes -= chunkLength;
+ inputCursor += chunkLength;
+ }
+ }
+
+ @Override
+ public void flush() throws IOException
+ {
+ try {
+ writeCompressedBlock();
+ } finally {
+ outputStream.flush();
+ }
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ try {
+ flush();
+ } finally {
+ outputStream.close();
+ }
+ }
+
+ /**
+ * Compress and write the current block to the OutputStream
+ */
+ private void writeCompressedBlock() throws IOException
+ {
+ final byte[] compressedBytes = LZFEncoder.encode(outputBuffer, position);
+ outputStream.write(compressedBytes);
+ position = 0;
+ }
+}
View
99 src/test/java/com/ning/compress/lzf/TestLZFInputStream.java
@@ -0,0 +1,99 @@
+package com.ning.compress.lzf;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.SecureRandom;
+
+import org.testng.Assert;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+public class TestLZFInputStream
+{
+ private static int BUFFER_SIZE = LZFChunk.MAX_CHUNK_LEN * 64;
+ private byte[] nonEncodableBytesToWrite = new byte[BUFFER_SIZE];
+ private byte[] bytesToWrite = new byte[BUFFER_SIZE];
+ private ByteArrayOutputStream nonCompressed;
+ private ByteArrayOutputStream compressed;
+
+ @BeforeTest(alwaysRun = true)
+ public void setUp() throws Exception
+ {
+ SecureRandom.getInstance("SHA1PRNG").nextBytes(nonEncodableBytesToWrite);
+ String phrase = "all work and no play make Jack a dull boy";
+ byte[] bytes = phrase.getBytes();
+ int cursor = 0;
+ while(cursor <= bytesToWrite.length) {
+ System.arraycopy(bytes, 0, bytesToWrite, cursor, (bytes.length+cursor < bytesToWrite.length)?bytes.length:bytesToWrite.length-cursor);
+ cursor += bytes.length;
+ }
+ nonCompressed = new ByteArrayOutputStream();
+ OutputStream os = new LZFOutputStream(nonCompressed);
+ os.write(nonEncodableBytesToWrite);
+ os.close();
+
+ compressed = new ByteArrayOutputStream();
+ os = new LZFOutputStream(compressed);
+ os.write(bytesToWrite);
+ os.close();
+ }
+
+ @Test
+ public void testDecompressNonEncodableReadByte() throws IOException
+ {
+ doDecompressReadBlock(nonCompressed.toByteArray(), nonEncodableBytesToWrite);
+ }
+
+ @Test
+ public void testDecompressNonEncodableReadBlock() throws IOException
+ {
+ doDecompressReadBlock(nonCompressed.toByteArray(), nonEncodableBytesToWrite);
+ }
+
+ @Test
+ public void testDecompressEncodableReadByte() throws IOException
+ {
+ doDecompressReadBlock(compressed.toByteArray(), bytesToWrite);
+ }
+
+ @Test
+ public void testDecompressEncodableReadBlock() throws IOException
+ {
+ doDecompressReadBlock(compressed.toByteArray(), bytesToWrite);
+ }
+
+ public void doDecompressNonEncodableReadByte(byte[] bytes, byte[] reference) throws IOException
+ {
+ ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
+ int outputBytes = 0;
+ InputStream is = new LZFInputStream(bis);
+ int val;
+ while((val=is.read()) != -1) {
+ byte testVal = (byte)(val & 255);
+ Assert.assertTrue(testVal == reference[outputBytes]);
+ outputBytes++;
+ }
+ Assert.assertTrue(outputBytes == reference.length);
+ }
+
+
+ private void doDecompressReadBlock(byte[] bytes, byte[] reference) throws IOException
+ {
+ ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
+ int outputBytes = 0;
+ InputStream is = new LZFInputStream(bis);
+ int val;
+ byte[] buffer = new byte[65536+23];
+ while((val=is.read(buffer)) != -1) {
+ for(int i = 0; i < val; i++) {
+ byte testVal = buffer[i];
+ Assert.assertTrue(testVal == reference[outputBytes]);
+ outputBytes++;
+ }
+ }
+ Assert.assertTrue(outputBytes == reference.length);
+ }
+}
View
79 src/test/java/com/ning/compress/lzf/TestLZFOutputStream.java
@@ -0,0 +1,79 @@
+package com.ning.compress.lzf;
+
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
+import java.security.SecureRandom;
+
+import org.testng.Assert;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+public class TestLZFOutputStream {
+
+ private static int BUFFER_SIZE = LZFChunk.MAX_CHUNK_LEN * 64;
+ private byte[] nonEncodableBytesToWrite = new byte[BUFFER_SIZE];
+ private byte[] bytesToWrite = new byte[BUFFER_SIZE];
+
+ @BeforeTest(alwaysRun = true)
+ public void setUp() throws Exception
+ {
+ SecureRandom.getInstance("SHA1PRNG").nextBytes(nonEncodableBytesToWrite);
+ String phrase = "all work and no play make Jack a dull boy";
+ byte[] bytes = phrase.getBytes();
+ int cursor = 0;
+ while(cursor <= bytesToWrite.length) {
+ System.arraycopy(bytes, 0, bytesToWrite, cursor, (bytes.length+cursor < bytesToWrite.length)?bytes.length:bytesToWrite.length-cursor);
+ cursor += bytes.length;
+ }
+ }
+
+ @Test
+ public void testUnencodable() throws Exception
+ {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ OutputStream os = new LZFOutputStream(bos);
+ os.write(nonEncodableBytesToWrite);
+ os.close();
+ Assert.assertTrue(bos.toByteArray().length > nonEncodableBytesToWrite.length);
+ }
+
+ @Test
+ public void testStreaming() throws Exception
+ {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ OutputStream os = new LZFOutputStream(bos);
+ os.write(bytesToWrite);
+ os.close();
+ Assert.assertTrue(bos.toByteArray().length > 10);
+ Assert.assertTrue(bos.toByteArray().length < bytesToWrite.length*.5);
+ }
+
+ @Test
+ public void testSingleByte() throws Exception
+ {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ OutputStream os = new LZFOutputStream(bos);
+ for(int idx = 0; idx < BUFFER_SIZE; idx++) {
+ os.write(bytesToWrite[idx]);
+ if(idx % 1023 == 0) {
+ os.flush();
+ }
+ }
+ os.close();
+ Assert.assertTrue(bos.toByteArray().length > 10);
+ Assert.assertTrue(bos.toByteArray().length < bytesToWrite.length*.5);
+ }
+
+ @Test
+ public void testPartialBuffer() throws Exception
+ {
+ int offset = 255;
+ int len = 1<<17;
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ OutputStream os = new LZFOutputStream(bos);
+ os.write(bytesToWrite, offset, len);
+ os.close();
+ Assert.assertTrue(bos.toByteArray().length > 10);
+ Assert.assertTrue(bos.toByteArray().length < bytesToWrite.length*.5);
+ }
+}
Please sign in to comment.
Something went wrong with that request. Please try again.