Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

reuse decompressor to reduce memory consumption

  • Loading branch information...
commit 793a1323466f143e46fe4aff09ab7d2aacc7b7ca 1 parent 6bb1b7f
Yu Xu yuxutw authored
BIN  ivy/ivy-2.2.0.jar
View
Binary file not shown
12 src/java/com/hadoop/compression/lzo/LzopCodec.java
View
@@ -23,6 +23,7 @@
import java.io.InputStream;
import java.io.OutputStream;
+import org.apache.hadoop.io.compress.CodecPool;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionInputStream;
import org.apache.hadoop.io.compress.CompressionOutputStream;
@@ -86,7 +87,16 @@ public CompressionInputStream createInputStream(InputStream in,
@Override
public CompressionInputStream createInputStream(InputStream in) throws IOException {
- return createInputStream(in, createDecompressor());
+ // Ensure native-lzo library is loaded & initialized
+ if (!isNativeLzoLoaded(getConf())) {
+ throw new RuntimeException("native-lzo library not available");
+ }
+ /*create a decompressor and tell LzoInputStream to reuse it
+ * (return it to the pool when LzoInputStream is closed.
+ */
+
+ return new LzopInputStream(in, CodecPool.getDecompressor(this),
+ getConf().getInt(LZO_BUFFER_SIZE_KEY, DEFAULT_LZO_BUFFER_SIZE), true);
}
@Override
12 src/java/com/hadoop/compression/lzo/LzopInputStream.java
View
@@ -31,6 +31,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.compress.BlockDecompressorStream;
+import org.apache.hadoop.io.compress.CodecPool;
import org.apache.hadoop.io.compress.Decompressor;
public class LzopInputStream extends BlockDecompressorStream {
@@ -47,11 +48,18 @@
private int noUncompressedBytes = 0;
private int noCompressedBytes = 0;
private int uncompressedBlockSize = 0;
+ private boolean reuseDecompressor;
public LzopInputStream(InputStream in, Decompressor decompressor,
int bufferSize) throws IOException {
+ this(in, decompressor, bufferSize, false);
+ }
+
+
+ public LzopInputStream(InputStream in, Decompressor decompressor, int bufferSize, boolean reuseDecompressor) throws IOException {
super(in, decompressor, bufferSize);
readHeader(in);
+ this.reuseDecompressor = reuseDecompressor;
}
@@ -83,7 +91,7 @@ private static void readFully( InputStream in, byte buf[],
* Read len bytes into buf, st LSB of int returned is the last byte of the
* first word read.
*/
- private static int readInt(InputStream in, byte[] buf, int len)
+ private static int readInt(InputStream in, byte[] buf, int len)
throws IOException {
readFully(in, buf, 0, len);
int ret = (0xFF & buf[0]) << 24;
@@ -331,6 +339,8 @@ public void close() throws IOException {
decompressor.decompress(b, 0, b.length);
}
super.close();
+ if(reuseDecompressor)
+ CodecPool.returnDecompressor(decompressor);
try {
verifyChecksums();
} catch (IOException e) {
Please sign in to comment.
Something went wrong with that request. Please try again.