Merge branch 'master' of http://github.com/toddlipcon/hadoop-lzo into…

… toddlipcon-master
cloudera · Dec 15, 2010 · 53ded8e · 53ded8e
2 parents c7acdaa + fdbddca
commit 53ded8e
Show file tree

Hide file tree

Showing 9 changed files with 2,245 additions and 4,412 deletions.
diff --git a/CHANGES.txt b/CHANGES.txt
diff --git a/build.xml b/build.xml
@@ -18,7 +18,7 @@
  <http://www.gnu.org/licenses/>.
 -->
 
-<project name="Hadoop-GPL-Compression" default="jar" 
+<project name="Hadoop-GPL-Compression" default="compile"
    xmlns:ivy="antlib:org.apache.ivy.ant"> 
 
   <!-- Load all the default properties, and any the user wants    -->
@@ -28,7 +28,7 @@
 
   <property name="Name" value="Hadoop GPL Compression"/>
   <property name="name" value="hadoop-lzo"/>
-  <property name="version" value="0.4.6"/>
+  <property name="version" value="0.4.8"/>
   <property name="final.name" value="${name}-${version}"/>
   <property name="year" value="2008"/>
 
@@ -223,8 +223,15 @@
     </copy>
 
   </target>
+  <target name="check-native-uptodate">
+    <uptodate
+        property="native.uptodate"
+        targetfile="${build.native}/lib/libgplcompression.so">
+      <srcfiles dir="${native.src.dir}" includes="**/*" />
+    </uptodate>
+  </target>
 
-  <target name="compile-native" depends="compile-java">
+  <target name="compile-native" depends="compile-java,check-native-uptodate" unless="native.uptodate">
 
     <mkdir dir="${build.native}/lib"/>
     <mkdir dir="${build.native}/src/com/hadoop/compression/lzo"/>
@@ -258,7 +265,7 @@
 
   </target>
 
-  <target name="compile" depends="compile-java,buildinfo" 
+  <target name="compile" depends="compile-java,compile-native,buildinfo" 
           description="Compile all">
   </target>
 

diff --git a/src/java/com/hadoop/compression/lzo/LzoCompressor.java b/src/java/com/hadoop/compression/lzo/LzoCompressor.java
@@ -220,8 +220,24 @@ public LzoCompressor(CompressionStrategy strategy, int directBufferSize) {
   private void init(CompressionStrategy strategy, int directBufferSize) {
     this.strategy = strategy;
     this.directBufferSize = directBufferSize;
-    uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
-    compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+
+    // If the direct buffers are already allocated and the correct size
+    // don't reallocate - this avoids potential rampant heap growth since GC
+    // doesn't get triggered by native buffer space usage.
+    if (uncompressedDirectBuf == null ||
+        uncompressedDirectBuf.capacity() != directBufferSize) {
+      uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+    } else {
+      uncompressedDirectBuf.clear();
+    }
+
+    if (compressedDirectBuf == null ||
+        compressedDirectBuf.capacity() != directBufferSize) {
+      compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+    } else {
+      compressedDirectBuf.clear();
+    }
+
     compressedDirectBuf.position(directBufferSize);
     reset();
 

diff --git a/src/java/com/hadoop/compression/lzo/LzoDecompressor.java b/src/java/com/hadoop/compression/lzo/LzoDecompressor.java
@@ -296,21 +296,19 @@ public synchronized int decompress(byte[] b, int off, int len)
       }
 
       // Check if there is data to decompress
-      if (compressedDirectBufLen <= 0) {
-        return 0;
-      }
-
-      // Re-initialize the lzo's output direct-buffer
-      uncompressedDirectBuf.rewind();
-      uncompressedDirectBuf.limit(directBufferSize);
+      if (compressedDirectBufLen > 0) {
+        // Re-initialize the lzo's output direct-buffer
+        uncompressedDirectBuf.rewind();
+        uncompressedDirectBuf.limit(directBufferSize);
 
-      // Decompress data
-      numBytes = decompressBytesDirect(strategy.getDecompressor());
-      uncompressedDirectBuf.limit(numBytes);
+        // Decompress data
+        numBytes = decompressBytesDirect(strategy.getDecompressor());
+        uncompressedDirectBuf.limit(numBytes);
 
-      // Return atmost 'len' bytes
-      numBytes = Math.min(numBytes, len);
-      ((ByteBuffer)uncompressedDirectBuf).get(b, off, numBytes);
+        // Return atmost 'len' bytes
+        numBytes = Math.min(numBytes, len);
+        ((ByteBuffer)uncompressedDirectBuf).get(b, off, numBytes);
+      }
     }
 
     // Set 'finished' if lzo has consumed all user-data