DRILL-4237, DRILL-4478: Implement hash to use murmur3 and add corresp…

…ondent unit tests + Avoid object or extra buffer creation + Clean up tests closes #485
apache · Apr 20, 2016 · c6a03eb · c6a03eb
1 parent 3b056db
commit c6a03eb
Show file tree

Hide file tree

Showing 14 changed files with 638 additions and 262 deletions.
diff --git a/common/src/main/java/org/apache/drill/common/util/DrillStringUtils.java b/common/src/main/java/org/apache/drill/common/util/DrillStringUtils.java
@@ -163,10 +163,10 @@ private static void appendByte(StringBuilder result, byte b) {
   public static int parseBinaryString(ByteBuf str, int strStart, int strEnd) {
     int length = (strEnd - strStart);
     int dstEnd = strStart;
-    for (int i = strStart; i < length ; i++) {
+    for (int i = strStart; i < strStart+length ; i++) {
       byte b = str.getByte(i);
       if (b == '\\'
-          && length > i+3
+          && strEnd > i+3
           && (str.getByte(i+1) == 'x' || str.getByte(i+1) == 'X')) {
         // ok, take next 2 hex digits.
         byte hd1 = str.getByte(i+2);

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DrillHash.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/DrillHash.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.expr.fn.impl;
+
+import io.netty.buffer.DrillBuf;
+import io.netty.util.internal.PlatformDependent;
+
+/**
+ * The base class of hash classes used in Drill.
+ */
+public class DrillHash {
+
+    public static final long getLongLittleEndian(long offset) {
+        //return PlatformDependent.getLong(offset);
+        return     ((long) PlatformDependent.getByte(offset+7)    << 56)
+                | ((PlatformDependent.getByte(offset+6) & 0xffL) << 48)
+                | ((PlatformDependent.getByte(offset+5) & 0xffL) << 40)
+                | ((PlatformDependent.getByte(offset+4) & 0xffL) << 32)
+                | ((PlatformDependent.getByte(offset+3) & 0xffL) << 24)
+                | ((PlatformDependent.getByte(offset+2) & 0xffL) << 16)
+                | ((PlatformDependent.getByte(offset+1) & 0xffL) << 8)
+                | ((PlatformDependent.getByte(offset) & 0xffL));
+    }
+
+    public static final long getIntLittleEndian(long offset) {
+        long retl = 0;
+        retl = ((PlatformDependent.getByte(offset+3) &0xffL) << 24)
+                | ((PlatformDependent.getByte(offset+2) & 0xffL) << 16)
+                | ((PlatformDependent.getByte(offset+1) & 0xffL) << 8)
+                | ((PlatformDependent.getByte(offset) & 0xffL));
+        return retl;
+    }
+
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Hash32AsDouble.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Hash32AsDouble.java
@@ -45,6 +45,7 @@
  */
 public class Hash32AsDouble {
   @FunctionTemplate(name = "hash32AsDouble", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+
   public static class NullableFloatHash implements DrillSimpleFunc {
 
     @Param
@@ -59,7 +60,7 @@ public void eval() {
       if (in.isSet == 0) {
         out.value = 0;
       } else {
-        out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32((double) in.value, 0);
+        out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash32(in.value, 0);
       }
     }
   }
@@ -76,7 +77,7 @@ public void setup() {
     }
 
     public void eval() {
-      out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32((double) in.value, 0);
+      out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash32(in.value, 0);
     }
   }
 
@@ -95,7 +96,7 @@ public void eval() {
       if (in.isSet == 0) {
         out.value = 0;
       } else {
-        out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32((double) in.value, 0);
+        out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash32(in.value, 0);
       }
     }
   }
@@ -112,7 +113,7 @@ public void setup() {
     }
 
     public void eval() {
-      out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(in.value, 0);
+      out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash32(in.value, 0);
     }
   }
 
@@ -131,7 +132,7 @@ public void eval() {
       if (in.isSet == 0) {
         out.value = 0;
       } else {
-        out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32((double) in.value, 0);
+        out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash32((long) in.value, 0);
       }
     }
   }
@@ -150,7 +151,7 @@ public void eval() {
       if (in.isSet == 0) {
         out.value = 0;
       } else {
-        out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32((double) in.value, 0);
+        out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash32(in.value, 0);
       }
     }
   }
@@ -167,7 +168,7 @@ public void setup() {
     }
 
     public void eval() {
-      out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32((double) in.value, 0);
+      out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash32((long)in.value, 0);
     }
   }
 
@@ -182,7 +183,7 @@ public void setup() {
     }
 
     public void eval() {
-      out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32((double) in.value, 0);
+      out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash32(in.value, 0);
     }
   }
 
@@ -198,7 +199,7 @@ public void setup() {
 
     public void eval() {
       java.math.BigDecimal input = new java.math.BigDecimal(java.math.BigInteger.valueOf(in.value), in.scale);
-      out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(input.doubleValue(), 0);
+      out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash32(input.doubleValue(), 0);
     }
   }
 
@@ -217,7 +218,7 @@ public void eval() {
         out.value = 0;
       } else {
         java.math.BigDecimal input = new java.math.BigDecimal(java.math.BigInteger.valueOf(in.value), in.scale);
-        out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(input.doubleValue(), 0);
+        out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash32(input.doubleValue(), 0);
       }
     }
   }
@@ -234,7 +235,7 @@ public void setup() {
 
     public void eval() {
       java.math.BigDecimal input = new java.math.BigDecimal(java.math.BigInteger.valueOf(in.value), in.scale);
-      out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(input.doubleValue(), 0);
+      out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash32(input.doubleValue(), 0);
     }
   }
 
@@ -253,7 +254,7 @@ public void eval() {
         out.value = 0;
       } else {
         java.math.BigDecimal input = new java.math.BigDecimal(java.math.BigInteger.valueOf(in.value), in.scale);
-        out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(input.doubleValue(), 0);
+        out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash32(input.doubleValue(), 0);
       }
     }
   }
@@ -271,7 +272,7 @@ public void setup() {
     public void eval() {
       java.math.BigDecimal input = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(in.buffer,
           in.start, in.nDecimalDigits, in.scale);
-      out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(input.doubleValue(), 0);
+      out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash32(input.doubleValue(), 0);
     }
   }
 
@@ -291,7 +292,7 @@ public void eval() {
       } else {
         java.math.BigDecimal input = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(in.buffer,
             in.start, in.nDecimalDigits, in.scale);
-        out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(input.doubleValue(), 0);
+        out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash32(input.doubleValue(), 0);
       }
     }
   }
@@ -309,7 +310,7 @@ public void setup() {
     public void eval() {
       java.math.BigDecimal input = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(in.buffer,
           in.start, in.nDecimalDigits, in.scale);
-      out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(input.doubleValue(), 0);
+      out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash32(input.doubleValue(), 0);
     }
   }
 
@@ -329,7 +330,7 @@ public void eval() {
       } else {
         java.math.BigDecimal input = org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(in.buffer,
             in.start, in.nDecimalDigits, in.scale);
-        out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(input.doubleValue(), 0);
+        out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash32(input.doubleValue(), 0);
       }
     }
   }