Update to arrow 18.0.0 (#2856)

* Update to arrow/parquet 18.0.0 * clean up field creation API * Update use `BasicDecimalArray`
apache · Jul 11, 2022 · d0d5564 · d0d5564
1 parent b6370ee
commit d0d5564
Show file tree

Hide file tree

Showing 19 changed files with 35 additions and 34 deletions.
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
@@ -29,7 +29,7 @@ rust-version = "1.59"
 readme = "README.md"
 
 [dependencies]
-arrow = { version = "17.0.0" }
+arrow = { version = "18.0.0" }
 clap = { version = "3", features = ["derive", "cargo"] }
 datafusion = { path = "../datafusion/core", version = "9.0.0" }
 dirs = "4.0.0"

diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml
@@ -34,7 +34,7 @@ path = "examples/avro_sql.rs"
 required-features = ["datafusion/avro"]
 
 [dev-dependencies]
-arrow-flight = { version = "17.0.0" }
+arrow-flight = { version = "18.0.0" }
 async-trait = "0.1.41"
 datafusion = { path = "../datafusion/core" }
 futures = "0.3"

diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml
@@ -38,11 +38,11 @@ jit = ["cranelift-module"]
 pyarrow = ["pyo3"]
 
 [dependencies]
-arrow = { version = "17.0.0", features = ["prettyprint"] }
+arrow = { version = "18.0.0", features = ["prettyprint"] }
 avro-rs = { version = "0.13", features = ["snappy"], optional = true }
 cranelift-module = { version = "0.85.0", optional = true }
 object_store = { version = "0.3", optional = true }
 ordered-float = "3.0"
-parquet = { version = "17.0.0", features = ["arrow"], optional = true }
+parquet = { version = "18.0.0", features = ["arrow"], optional = true }
 pyo3 = { version = "0.16", optional = true }
 sqlparser = "0.18"
diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
@@ -55,7 +55,7 @@ unicode_expressions = ["datafusion-physical-expr/regex_expressions", "datafusion
 
 [dependencies]
 ahash = { version = "0.7", default-features = false }
-arrow = { version = "17.0.0", features = ["prettyprint"] }
+arrow = { version = "18.0.0", features = ["prettyprint"] }
 async-trait = "0.1.41"
 avro-rs = { version = "0.13", features = ["snappy"], optional = true }
 bytes = "1.1"
@@ -78,7 +78,7 @@ num_cpus = "1.13.0"
 object_store = "0.3.0"
 ordered-float = "3.0"
 parking_lot = "0.12"
-parquet = { version = "17.0.0", features = ["arrow", "async"] }
+parquet = { version = "18.0.0", features = ["arrow", "async"] }
 paste = "^1.0"
 pin-project-lite = "^0.2.7"
 pyo3 = { version = "0.16", optional = true }

diff --git a/datafusion/core/fuzz-utils/Cargo.toml b/datafusion/core/fuzz-utils/Cargo.toml
@@ -23,6 +23,6 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-arrow = { version = "17.0.0", features = ["prettyprint"] }
+arrow = { version = "18.0.0", features = ["prettyprint"] }
 env_logger = "0.9.0"
 rand = "0.8"
diff --git a/datafusion/core/src/physical_plan/hash_join.rs b/datafusion/core/src/physical_plan/hash_join.rs
@@ -22,11 +22,11 @@ use ahash::RandomState;
 
 use arrow::{
     array::{
-        as_dictionary_array, as_string_array, ArrayData, ArrayRef, BooleanArray,
-        Date32Array, Date64Array, DecimalArray, DictionaryArray, LargeStringArray,
-        PrimitiveArray, TimestampMicrosecondArray, TimestampMillisecondArray,
-        TimestampSecondArray, UInt32BufferBuilder, UInt32Builder, UInt64BufferBuilder,
-        UInt64Builder,
+        as_dictionary_array, as_string_array, ArrayData, ArrayRef, BasicDecimalArray,
+        BooleanArray, Date32Array, Date64Array, DecimalArray, DictionaryArray,
+        LargeStringArray, PrimitiveArray, TimestampMicrosecondArray,
+        TimestampMillisecondArray, TimestampSecondArray, UInt32BufferBuilder,
+        UInt32Builder, UInt64BufferBuilder, UInt64Builder,
     },
     compute,
     datatypes::{

diff --git a/datafusion/core/src/physical_plan/hash_utils.rs b/datafusion/core/src/physical_plan/hash_utils.rs
@@ -20,9 +20,9 @@
 use crate::error::{DataFusionError, Result};
 use ahash::{CallHasher, RandomState};
 use arrow::array::{
-    Array, ArrayRef, BooleanArray, Date32Array, Date64Array, DecimalArray,
-    DictionaryArray, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array,
-    Int8Array, LargeStringArray, StringArray, TimestampMicrosecondArray,
+    Array, ArrayRef, BasicDecimalArray, BooleanArray, Date32Array, Date64Array,
+    DecimalArray, DictionaryArray, Float32Array, Float64Array, Int16Array, Int32Array,
+    Int64Array, Int8Array, LargeStringArray, StringArray, TimestampMicrosecondArray,
     TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray,
     UInt16Array, UInt32Array, UInt64Array, UInt8Array,
 };

diff --git a/datafusion/core/src/physical_plan/join_utils.rs b/datafusion/core/src/physical_plan/join_utils.rs
@@ -170,8 +170,7 @@ fn output_join_field(old_field: &Field, join_type: &JoinType, is_left: bool) ->
     };
 
     if force_nullable {
-        // Could cleanup after https://github.com/apache/arrow-rs/issues/1934
-        Field::new(old_field.name(), old_field.data_type().clone(), true)
+        old_field.clone().with_nullable(true)
     } else {
         old_field.clone()
     }

diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml
@@ -36,6 +36,6 @@ path = "src/lib.rs"
 
 [dependencies]
 ahash = { version = "0.7", default-features = false }
-arrow = { version = "17.0.0", features = ["prettyprint"] }
+arrow = { version = "18.0.0", features = ["prettyprint"] }
 datafusion-common = { path = "../common", version = "9.0.0" }
 sqlparser = "0.18"
diff --git a/datafusion/jit/Cargo.toml b/datafusion/jit/Cargo.toml
@@ -36,7 +36,7 @@ path = "src/lib.rs"
 jit = []
 
 [dependencies]
-arrow = { version = "17.0.0" }
+arrow = { version = "18.0.0" }
 cranelift = "0.85.0"
 cranelift-jit = "0.85.0"
 cranelift-module = "0.85.0"

diff --git a/datafusion/optimizer/Cargo.toml b/datafusion/optimizer/Cargo.toml
@@ -37,7 +37,7 @@ default = ["unicode_expressions"]
 unicode_expressions = []
 
 [dependencies]
-arrow = { version = "17.0.0", features = ["prettyprint"] }
+arrow = { version = "18.0.0", features = ["prettyprint"] }
 async-trait = "0.1.41"
 chrono = { version = "0.4", default-features = false }
 datafusion-common = { path = "../common", version = "9.0.0" }

diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml
@@ -40,7 +40,7 @@ unicode_expressions = ["unicode-segmentation"]
 
 [dependencies]
 ahash = { version = "0.7", default-features = false }
-arrow = { version = "17.0.0", features = ["prettyprint"] }
+arrow = { version = "18.0.0", features = ["prettyprint"] }
 blake2 = { version = "^0.10.2", optional = true }
 blake3 = { version = "1.0", optional = true }
 chrono = { version = "0.4", default-features = false }

diff --git a/datafusion/physical-expr/src/aggregate/min_max.rs b/datafusion/physical-expr/src/aggregate/min_max.rs
@@ -26,10 +26,11 @@ use arrow::compute;
 use arrow::datatypes::{DataType, TimeUnit};
 use arrow::{
     array::{
-        ArrayRef, Date32Array, Date64Array, Float32Array, Float64Array, Int16Array,
-        Int32Array, Int64Array, Int8Array, LargeStringArray, StringArray,
-        TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
-        TimestampSecondArray, UInt16Array, UInt32Array, UInt64Array, UInt8Array,
+        ArrayRef, BasicDecimalArray, Date32Array, Date64Array, Float32Array,
+        Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, LargeStringArray,
+        StringArray, TimestampMicrosecondArray, TimestampMillisecondArray,
+        TimestampNanosecondArray, TimestampSecondArray, UInt16Array, UInt32Array,
+        UInt64Array, UInt8Array,
     },
     datatypes::Field,
 };

diff --git a/datafusion/physical-expr/src/aggregate/sum.rs b/datafusion/physical-expr/src/aggregate/sum.rs
@@ -26,8 +26,8 @@ use arrow::compute;
 use arrow::datatypes::DataType;
 use arrow::{
     array::{
-        ArrayRef, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array,
-        Int8Array, UInt16Array, UInt32Array, UInt64Array, UInt8Array,
+        ArrayRef, BasicDecimalArray, Float32Array, Float64Array, Int16Array, Int32Array,
+        Int64Array, Int8Array, UInt16Array, UInt32Array, UInt64Array, UInt8Array,
     },
     datatypes::Field,
 };

diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs
@@ -161,9 +161,9 @@ mod tests {
     use crate::expressions::col;
     use arrow::{
         array::{
-            Array, DecimalArray, Float32Array, Float64Array, Int16Array, Int32Array,
-            Int64Array, Int8Array, StringArray, Time64NanosecondArray,
-            TimestampNanosecondArray, UInt32Array,
+            Array, BasicDecimalArray, DecimalArray, Float32Array, Float64Array,
+            Int16Array, Int32Array, Int64Array, Int8Array, StringArray,
+            Time64NanosecondArray, TimestampNanosecondArray, UInt32Array,
         },
         datatypes::*,
         util::decimal::{BasicDecimal, Decimal128},

diff --git a/datafusion/physical-expr/src/expressions/try_cast.rs b/datafusion/physical-expr/src/expressions/try_cast.rs
@@ -119,7 +119,8 @@ mod tests {
     use super::*;
     use crate::expressions::col;
     use arrow::array::{
-        DecimalArray, DecimalBuilder, StringArray, Time64NanosecondArray,
+        BasicDecimalArray, DecimalArray, DecimalBuilder, StringArray,
+        Time64NanosecondArray,
     };
     use arrow::util::decimal::{BasicDecimal, Decimal128};
     use arrow::{

diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml
@@ -35,7 +35,7 @@ path = "src/lib.rs"
 [features]
 
 [dependencies]
-arrow = { version = "17.0.0" }
+arrow = { version = "18.0.0" }
 datafusion = { path = "../core", version = "9.0.0" }
 datafusion-common = { path = "../common", version = "9.0.0" }
 datafusion-expr = { path = "../expr", version = "9.0.0" }

diff --git a/datafusion/row/Cargo.toml b/datafusion/row/Cargo.toml
@@ -37,7 +37,7 @@ path = "src/lib.rs"
 jit = ["datafusion-jit"]
 
 [dependencies]
-arrow = { version = "17.0.0" }
+arrow = { version = "18.0.0" }
 datafusion-common = { path = "../common", version = "9.0.0" }
 datafusion-jit = { path = "../jit", version = "9.0.0", optional = true }
 paste = "^1.0"

diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.toml
@@ -38,7 +38,7 @@ unicode_expressions = []
 
 [dependencies]
 ahash = { version = "0.7", default-features = false }
-arrow = { version = "17.0.0", features = ["prettyprint"] }
+arrow = { version = "18.0.0", features = ["prettyprint"] }
 datafusion-common = { path = "../common", version = "9.0.0" }
 datafusion-expr = { path = "../expr", version = "9.0.0" }
 hashbrown = "0.12"