Change TupleDomainOrcPredicate to use UTF8 slices

prestodb · Mar 30, 2015 · b09336b · b09336b
1 parent 2e34244
commit b09336b
Show file tree

Hide file tree

Showing 6 changed files with 22 additions and 15 deletions.
diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/TupleDomainOrcPredicate.java b/presto-orc/src/main/java/com/facebook/presto/orc/TupleDomainOrcPredicate.java
@@ -28,7 +28,6 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.primitives.Primitives;
 import io.airlift.slice.Slice;
-import io.airlift.slice.Slices;
 
 import java.util.List;
 import java.util.Map;
@@ -115,7 +114,7 @@ else if (boxedJavaType == Double.class && columnStatistics.getDoubleStatistics()
             return createDomain(boxedJavaType, hasNullValue, columnStatistics.getDoubleStatistics());
         }
         else if (boxedJavaType == Slice.class && columnStatistics.getStringStatistics() != null) {
-            return createDomain(boxedJavaType, hasNullValue, columnStatistics.getStringStatistics(), Slices::utf8Slice);
+            return createDomain(boxedJavaType, hasNullValue, columnStatistics.getStringStatistics());
         }
         return Domain.create(SortedRangeSet.all(boxedJavaType), hasNullValue);
     }

diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/DwrfMetadataReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/DwrfMetadataReader.java
@@ -32,6 +32,7 @@
 import static com.facebook.presto.orc.metadata.CompressionKind.ZLIB;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
+import static io.airlift.slice.Slices.utf8Slice;
 
 public class DwrfMetadataReader
         implements MetadataReader
@@ -222,8 +223,8 @@ private static StringStatistics toStringStatistics(OrcProto.StringStatistics str
         }
 
         return new StringStatistics(
-                stringStatistics.hasMinimum() ? stringStatistics.getMinimum() : null,
-                stringStatistics.hasMaximum() ? stringStatistics.getMaximum() : null);
+                stringStatistics.hasMinimum() ? utf8Slice(stringStatistics.getMinimum()) : null,
+                stringStatistics.hasMaximum() ? utf8Slice(stringStatistics.getMaximum()) : null);
     }
 
     private static OrcType toType(OrcProto.Type type)

diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/OrcMetadataReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/OrcMetadataReader.java
@@ -31,6 +31,7 @@
 import static com.facebook.presto.orc.metadata.CompressionKind.UNCOMPRESSED;
 import static com.facebook.presto.orc.metadata.CompressionKind.ZLIB;
 import static com.google.common.base.Preconditions.checkState;
+import static io.airlift.slice.Slices.utf8Slice;
 
 public class OrcMetadataReader
         implements MetadataReader
@@ -226,8 +227,8 @@ private static StringStatistics toStringStatistics(OrcProto.StringStatistics str
         }
 
         return new StringStatistics(
-                stringStatistics.hasMinimum() ? stringStatistics.getMinimum() : null,
-                stringStatistics.hasMaximum() ? stringStatistics.getMaximum() : null);
+                stringStatistics.hasMinimum() ? utf8Slice(stringStatistics.getMinimum()) : null,
+                stringStatistics.hasMaximum() ? utf8Slice(stringStatistics.getMaximum()) : null);
     }
 
     private static DateStatistics toDateStatistics(OrcProto.DateStatistics dateStatistics, boolean isRowGroup)

diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/StringStatistics.java b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/StringStatistics.java
@@ -13,26 +13,28 @@
  */
 package com.facebook.presto.orc.metadata;
 
+import io.airlift.slice.Slice;
+
 public class StringStatistics
-        implements RangeStatistics<String>
+        implements RangeStatistics<Slice>
 {
-    private final String minimum;
-    private final String maximum;
+    private final Slice minimum;
+    private final Slice maximum;
 
-    public StringStatistics(String minimum, String maximum)
+    public StringStatistics(Slice minimum, Slice maximum)
     {
         this.minimum = minimum;
         this.maximum = maximum;
     }
 
     @Override
-    public String getMin()
+    public Slice getMin()
     {
         return minimum;
     }
 
     @Override
-    public String getMax()
+    public Slice getMax()
     {
         return maximum;
     }

diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestTupleDomainOrcPredicate.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestTupleDomainOrcPredicate.java
@@ -23,6 +23,7 @@
 import com.facebook.presto.spi.Range;
 import com.facebook.presto.spi.SortedRangeSet;
 import io.airlift.slice.Slice;
+import io.airlift.slice.Slices;
 import org.testng.annotations.Test;
 
 import static com.facebook.presto.orc.TupleDomainOrcPredicate.getDomain;
@@ -164,7 +165,9 @@ public void testString()
 
     private static ColumnStatistics stringColumnStats(Long numberOfValues, String minimum, String maximum)
     {
-        return new ColumnStatistics(numberOfValues, null, null, null, new StringStatistics(minimum, maximum), null);
+        Slice minSlice = (minimum == null ? null : Slices.utf8Slice(minimum));
+        Slice maxSlice = (maximum == null ? null : Slices.utf8Slice(maximum));
+        return new ColumnStatistics(numberOfValues, null, null, null, new StringStatistics(minSlice, maxSlice), null);
     }
 
     @Test

diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestingOrcPredicate.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestingOrcPredicate.java
@@ -16,6 +16,7 @@
 import com.facebook.presto.orc.metadata.ColumnStatistics;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Ordering;
+import io.airlift.slice.Slices;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
@@ -269,12 +270,12 @@ protected boolean chunkMatchesStats(List<String> chunk, ColumnStatistics columnS
             // statistics can be missing for any reason
             if (columnStatistics.getStringStatistics() != null) {
                 // verify min
-                if (!columnStatistics.getStringStatistics().getMin().equals(Ordering.natural().nullsLast().min(chunk))) {
+                if (!columnStatistics.getStringStatistics().getMin().equals(Slices.utf8Slice(Ordering.natural().nullsLast().min(chunk)))) {
                     return false;
                 }
 
                 // verify max
-                if (!columnStatistics.getStringStatistics().getMax().equals(Ordering.natural().nullsFirst().max(chunk))) {
+                if (!columnStatistics.getStringStatistics().getMax().equals(Slices.utf8Slice(Ordering.natural().nullsFirst().max(chunk)))) {
                     return false;
                 }
             }