Fix writing Unicode data in Raptor

prestodb · Jun 18, 2015 · 6357ea3 · 6357ea3
1 parent 32a7bdd
commit 6357ea3
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 6 deletions.
diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/Row.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/Row.java
@@ -16,12 +16,12 @@
 import com.facebook.presto.spi.Page;
 import com.facebook.presto.spi.block.Block;
 import com.facebook.presto.spi.type.Type;
-import com.facebook.presto.spi.type.VarcharType;
 import io.airlift.slice.Slice;
 
 import java.util.ArrayList;
 import java.util.List;
 
+import static com.facebook.presto.spi.type.VarcharType.VARCHAR;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Verify.verify;
 import static io.airlift.slice.SizeOf.SIZE_OF_BYTE;
@@ -73,9 +73,9 @@ else if (type.getJavaType() == double.class) {
                 rowBuilder.add(type.getDouble(block, position), SIZE_OF_DOUBLE);
             }
             else if (type.getJavaType() == Slice.class) {
-                byte[] bytes = type.getSlice(block, position).getBytes();
+                Slice slice = type.getSlice(block, position);
-                Object value = type.equals(VarcharType.VARCHAR) ? new String(bytes) : bytes;
+                Object value = type.equals(VARCHAR) ? slice.toStringUtf8() : slice.getBytes();
-                rowBuilder.add(value, bytes.length);
+                rowBuilder.add(value, slice.length());
             }
             else {
                 throw new AssertionError("unimplemented type: " + type);

diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardWriter.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardWriter.java
@@ -77,7 +77,7 @@ public void testWriter()
         RowPagesBuilder rowPagesBuilder = RowPagesBuilder.rowPagesBuilder(columnTypes)
                 .row(123, "hello", wrappedBuffer(bytes1), 123.456, true)
                 .row(null, "world", null, Double.POSITIVE_INFINITY, null)
-                .row(456, "bye", wrappedBuffer(bytes3), Double.NaN, false);
+                .row(456, "bye \u2603", wrappedBuffer(bytes3), Double.NaN, false);
 
         try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(new EmptyClassLoader());
              OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file)) {
@@ -104,7 +104,7 @@ public void testWriter()
             reader.readVector(1, stringVector);
             assertEquals(stringVector.vector[0], utf8Slice("hello"));
             assertEquals(stringVector.vector[1], utf8Slice("world"));
-            assertEquals(stringVector.vector[2], utf8Slice("bye"));
+            assertEquals(stringVector.vector[2], utf8Slice("bye \u2603"));
 
             SliceVector sliceVector = new SliceVector(3);
             reader.readVector(2, sliceVector);

diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestDistributedQueries.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestDistributedQueries.java
@@ -127,6 +127,11 @@ public void testCreateTableAsSelect()
                 "test_limit",
                 "SELECT orderkey FROM orders ORDER BY orderkey LIMIT 10",
                 "SELECT 10");
+
+        assertCreateTable(
+                "test_unicode",
+                "SELECT '\u2603' unicode",
+                "SELECT 1");
     }
 
     @Test