Fixes #174

techascent · Nov 24, 2020 · 4cb1cb3 · 4cb1cb3
1 parent 029e3c5
commit 4cb1cb3
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 0 deletions.
diff --git a/scripts/uuid_arrow.r b/scripts/uuid_arrow.r
@@ -0,0 +1,9 @@
+#!/usr/bin/Rscript
+
+library(dplyr)
+library(arrow)
+library(uuid)
+
+df=data_frame(uuids=rep(UUIDgenerate(), 10))
+
+arrow::write_ipc_stream(df, "test/data/uuid.arrow")
diff --git a/src/tech/v3/libs/arrow/schema.clj b/src/tech/v3/libs/arrow/schema.clj
@@ -71,6 +71,7 @@
                  (ft-fn (.getIndexType encoding) encoding)
                  ;;If no encoding is provided then just save the string as text
                  (ft-fn (ArrowType$Utf8.)))
+       :uuid (ft-fn (ArrowType$Utf8.))
        :text (ft-fn (ArrowType$Utf8.))
        :encoded-text (ft-fn (ArrowType$Utf8.))))))
 

diff --git a/test/tech/v3/libs/arrow_test.clj b/test/tech/v3/libs/arrow_test.clj
@@ -119,6 +119,24 @@
            (vec (first (ds/columns inp-data)))))))
 
 
+(deftest uuid-test
+  (try
+    (let [uuid-ds (ds/->dataset "test/data/uuid.parquet"
+                                {:parser-fn {"uuids" :uuid}})
+          _ (arrow/write-dataset-to-stream! uuid-ds "test-uuid.arrow")
+          copying-ds (arrow/read-stream-dataset-copying "test-uuid.arrow")
+          inplace-ds (arrow/read-stream-dataset-inplace "test-uuid.arrow")]
+      (is (= :text ((comp :datatype meta) (copying-ds "uuids"))))
+      (is (= :text ((comp :datatype meta) (inplace-ds "uuids"))))
+      (is (= (vec (copying-ds "uuids"))
+             (vec (inplace-ds "uuids"))))
+      (is (= (mapv str (uuid-ds "uuids"))
+             (mapv str (copying-ds "uuids")))))
+    (finally
+      (.delete (java.io.File. "test-uuid.arrow")))))
+
+
+
 (comment
 
   (deftest big-arrow-text