1. Added server config file support to CommonCrawlServer. 2. Fixed mi…

…ssing offset bug in TextBytes. 3. Add RawComparator,RawComparable support to FlexBuffer.

src/org/commoncrawl/server/CommonCrawlServer.java

@@ -194,6 +194,10 @@ public void setAsyncWebDispatch(boolean asyncWebDispatch) {
     _useAsyncWebDispatch = asyncWebDispatch;
   }
 
+  public FileSystem getFileSystem() { 
+    return _fileSystem;
+  }
+
   public String getHostName() {
     return _hostName;
   }
@@ -495,6 +499,10 @@ public static void main(String argv[]) throws Exception {
         printCommonUsage();
         return;
       }
+      if (_commonConfig._configName != null) { 
+        LOG.info("Processing Config File:" + _commonConfig._configName);
+        conf.addResource(_commonConfig._configName);
+      }
 
       if (_commonConfig._hostName != null) {
         conf.set("org.commoncrawl.hostname", _commonConfig._hostName);

src/org/commoncrawl/util/shared/FlexBuffer.java

@@ -21,8 +21,11 @@
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.RawComparator;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.io.file.tfile.RawComparable;
 
 /**
  * A tweaking of the hadoop Buffer class, basically allowing for cheap buffer
@@ -34,7 +37,7 @@
  * capacity.
  * 
  */
-public final class FlexBuffer implements WritableComparable, Cloneable {
+public final class FlexBuffer implements WritableComparable<FlexBuffer>,RawComparator<FlexBuffer>,RawComparable, Cloneable {
   /** Number of valid bytes in this.bytes. */
   int     count;
   /** Backing store for Buffer. */
@@ -295,26 +298,16 @@ public int hashCode() {
    * @return Positive if this is bigger than other, 0 if they are equal, and
    *         negative if this is smaller than other.
    */
-  public int compareTo(Object other) {
-    FlexBuffer right = ((FlexBuffer) other);
-    byte[] lb = this.get();
-    int leftOffset = this.getOffset();
-    byte[] rb = right.get();
-    int rightOffset = right.getOffset();
-    for (int i = 0; i < count && i < right.count; i++) {
-      int a = (lb[i + leftOffset] & 0xff);
-      int b = (rb[i + rightOffset] & 0xff);
-      if (a != b) {
-        return a - b;
-      }
-    }
-    return count - right.count;
+  public int compareTo(FlexBuffer other) {
+    return BytesWritable.Comparator.compareBytes(
+        this.zbytes, this.offset, this.count,
+        other.zbytes, other.offset, other.count);
   }
 
   // inherit javadoc
   public boolean equals(Object other) {
     if (other instanceof FlexBuffer && this != other) {
-      return compareTo(other) == 0;
+      return compareTo((FlexBuffer)other) == 0;
     }
     return (this == other);
   }
@@ -393,4 +386,29 @@ public void write(DataOutput out) throws IOException {
       out.write(get(), getOffset(), getCount());
     }
   }
+
+  @Override
+  public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+    return BytesWritable.Comparator.compareBytes(b1,s1,l1,b2,s2,l2);
+  }
+
+  @Override
+  public int compare(FlexBuffer o1, FlexBuffer o2) {
+    return o1.compareTo(o2);
+  }
+
+  @Override
+  public byte[] buffer() {
+    return zbytes;
+  }
+
+  @Override
+  public int offset() {
+    return offset;
+  }
+
+  @Override
+  public int size() {
+    return count;
+  }
 }

src/org/commoncrawl/util/shared/TextBytes.java

@@ -260,12 +260,12 @@ public void set(byte[] utf8,boolean shared) {
 
   /** copy a text. */
   public void set(Text other,boolean shared) {
-    set(other.getBytes(), 0, other.getLength(),shared);
+    set(other.getBytes(), 0 , other.getLength(),shared);
   }
 
   /** copy a textbytes. */
   public void set(TextBytes other,boolean shared) {
-    set(other.getBytes(), 0, other.getLength(),shared);
+    set(other.getBytes(), other.getOffset(), other.getLength(),shared);
   }
 
   /**