Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

handle differences in CodecPool.getCompressor()

interface between CDH3 and Apache Hadoop.
  • Loading branch information...
commit a39bacff46a38eb712015d4607b14cb875537722 1 parent 9ab0565
@rangadi rangadi authored
Showing with 29 additions and 2 deletions.
  1. +29 −2 src/java/com/hadoop/compression/lzo/LzopCodec.java
View
31 src/java/com/hadoop/compression/lzo/LzopCodec.java
@@ -23,6 +23,7 @@
import java.io.InputStream;
import java.io.OutputStream;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.compress.CodecPool;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionInputStream;
@@ -46,11 +47,27 @@
public static final int LZOP_COMPAT_VERSION = 0x0940;
public static final String DEFAULT_LZO_EXTENSION = ".lzo";
+ /**
+ * CodecPool.getCompressor() that takes conf is supported only in CDH3.
+ * The change is yet to make it to Apache Hadoop. Fall back to old
+ * getCompressor() if the new interface is not present.
+ */
+ private static boolean codecPoolSupportsConf = false;
+ static {
+ try {
+ codecPoolSupportsConf =
+ null != CodecPool.class.getMethod("getCompressor",
+ CompressionCodec.class,
+ Configuration.class);
+ } catch (Exception e) {
+ }
+ }
+
@Override
public CompressionOutputStream createOutputStream(OutputStream out) throws IOException {
//get a compressor which will be returned to the pool when the output stream
//is closed.
- return createOutputStream(out, CodecPool.getCompressor(this, getConf()));
+ return createOutputStream(out, getCompressor());
}
public CompressionOutputStream createIndexedOutputStream(OutputStream out,
@@ -58,7 +75,7 @@ public CompressionOutputStream createIndexedOutputStream(OutputStream out,
throws IOException {
//get a compressor which will be returned to the pool when the output stream
//is closed.
- return createIndexedOutputStream(out, indexOut, CodecPool.getCompressor(this, getConf()));
+ return createIndexedOutputStream(out, indexOut, getCompressor());
}
@Override
@@ -113,6 +130,16 @@ public Decompressor createDecompressor() {
return new LzopDecompressor(getConf().getInt(LZO_BUFFER_SIZE_KEY, DEFAULT_LZO_BUFFER_SIZE));
}
+ private Compressor getCompressor() {
+ if (codecPoolSupportsConf) {
+ return CodecPool.getCompressor(this, getConf());
+ } else {
+ // this is potentially wrong since user's configuration changes between
+ // different two instances of LzopCodec are not honored.
+ return CodecPool.getCompressor(this);
+ }
+ }
+
@Override
public String getDefaultExtension() {
return DEFAULT_LZO_EXTENSION;
Please sign in to comment.
Something went wrong with that request. Please try again.