Some modifications to MMapUtils and corresponding changes to RiceCode…

…r. Also disable native compilation in build.xml by default.
commoncrawl · Nov 14, 2011 · 1deb913 · 1deb913
1 parent 1eb8b37
commit 1deb913
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 9 deletions.
diff --git a/src/org/commoncrawl/util/shared/MMapUtils.java b/src/org/commoncrawl/util/shared/MMapUtils.java
@@ -119,6 +119,8 @@ public static class MMapFile {
     long length = -1;
     ByteBuffer buffers[]=null;
     int        bufSizes[] = null;
+    private int refCount =0;
+    private boolean closePending = false;
 
     public MMapFile(File input) throws IOException { 
       RandomAccessFile raf = new RandomAccessFile(input, "r");
@@ -159,9 +161,31 @@ public FSDataInputStream newInputStream() throws IOException {
       return dataStream;
     }
 
-    public void close()throws IOException { 
-      for (ByteBuffer buffer : buffers) { 
-        cleanMapping(buffer);
+    private synchronized void addRef() { 
+      refCount++;
+    }
+
+    private synchronized void release() { 
+      if (--refCount == 0 && closePending) { 
+        try {
+          close();
+        } catch (IOException e) {
+          LOG.error(CCStringUtils.stringifyException(e));
+        }
+      }
+    }
+
+    public synchronized void close()throws IOException {
+      if (refCount == 0) {
+        if (buffers != null) { 
+          for (ByteBuffer buffer : buffers) { 
+            cleanMapping(buffer);
+          }
+        }
+        buffers = null;
+      }
+      else { 
+        closePending = true;
       }
     }
 
@@ -180,6 +204,7 @@ public class MMapFileInputStream extends FSInputStream {
       private ByteBuffer curBuf; // redundant for speed: buffers[curBufIndex]
 
       public MMapFileInputStream() throws IOException {
+        addRef();
         seek(0L);
       }
 
@@ -223,6 +248,11 @@ public int read(byte[] bytes, int offset, int len) throws IOException {
         }
       }
 
+      @Override
+      public void close() throws IOException {
+        release();
+      }
+
       @Override
       public int available() throws IOException {
         long amtAvailable = (length() - getPos());

diff --git a/src/org/commoncrawl/util/shared/RiceCoder.java b/src/org/commoncrawl/util/shared/RiceCoder.java
@@ -1,19 +1,13 @@
 package org.commoncrawl.util.shared;
 
-import java.io.DataOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.zip.GZIPOutputStream;
 
 import junit.framework.Assert;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSInputStream;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.io.WritableUtils;
 
 /**
  * Encode a set of Longs using Rice coding, a variant of Golomb coding,