Update to arrow 31

benchmarks/Cargo.toml

@@ -33,14 +33,14 @@ simd = ["datafusion/simd"]
 snmalloc = ["snmalloc-rs"]
 
 [dependencies]
-arrow = "30.0.1"
+arrow = "31.0.0"
 datafusion = { path = "../datafusion/core", version = "16.0.0", features = ["scheduler"] }
 env_logger = "0.10"
 futures = "0.3"
 mimalloc = { version = "0.1", optional = true, default-features = false }
 num_cpus = "1.13.0"
 object_store = "0.5.0"
-parquet = "30.0.1"
+parquet = "31.0.0"
 parquet-test-utils = { path = "../parquet-test-utils/", version = "0.1.0" }
 rand = "0.8.4"
 serde = { version = "1.0.136", features = ["derive"] }

datafusion-cli/Cargo.toml

@@ -29,7 +29,7 @@ rust-version = "1.62"
 readme = "README.md"
 
 [dependencies]
-arrow = "30.0.1"
+arrow = "31.0.0"
 async-trait = "0.1.41"
 clap = { version = "3", features = ["derive", "cargo"] }
 datafusion = { path = "../datafusion/core", version = "16.0.0" }

datafusion-examples/Cargo.toml

@@ -34,8 +34,8 @@ path = "examples/avro_sql.rs"
 required-features = ["datafusion/avro"]
 
 [dev-dependencies]
-arrow = "30.0.1"
-arrow-flight = "30.0.1"
+arrow = "31.0.0"
+arrow-flight = "31.0.0"
 async-trait = "0.1.41"
 datafusion = { path = "../datafusion/core" }
 datafusion-common = { path = "../datafusion/common" }

datafusion/common/Cargo.toml

@@ -40,11 +40,11 @@ pyarrow = ["pyo3", "arrow/pyarrow"]
 
 [dependencies]
 apache-avro = { version = "0.14", default-features = false, features = ["snappy"], optional = true }
-arrow = { version = "30.0.1", default-features = false }
+arrow = { version = "31.0.0", default-features = false }
 chrono = { version = "0.4", default-features = false }
 cranelift-module = { version = "0.89.0", optional = true }
 num_cpus = "1.13.0"
 object_store = { version = "0.5.0", default-features = false, optional = true }
-parquet = { version = "30.0.1", default-features = false, optional = true }
+parquet = { version = "31.0.0", default-features = false, optional = true }
 pyo3 = { version = "0.17.1", optional = true }
 sqlparser = "0.30"

datafusion/core/Cargo.toml

@@ -57,7 +57,7 @@ unicode_expressions = ["datafusion-physical-expr/regex_expressions", "datafusion
 [dependencies]
 ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
 apache-avro = { version = "0.14", optional = true }
-arrow = { version = "30.0.1", features = ["prettyprint"] }
+arrow = { version = "31.0.0", features = ["prettyprint"] }
 async-compression = { version = "0.3.14", features = ["bzip2", "gzip", "xz", "futures-io", "tokio"], optional = true }
 async-trait = "0.1.41"
 bytes = "1.1"
@@ -83,7 +83,7 @@ num-traits = { version = "0.2", optional = true }
 num_cpus = "1.13.0"
 object_store = "0.5.0"
 parking_lot = "0.12"
-parquet = { version = "30.0.1", features = ["arrow", "async"] }
+parquet = { version = "31.0.0", features = ["arrow", "async"] }
 paste = "^1.0"
 percent-encoding = "2.2.0"
 pin-project-lite = "^0.2.7"
@@ -102,7 +102,7 @@ xz2 = { version = "0.1", optional = true }
 
 
 [dev-dependencies]
-arrow = { version = "30.0.1", features = ["prettyprint", "dyn_cmp_dict"] }
+arrow = { version = "31.0.0", features = ["prettyprint", "dyn_cmp_dict"] }
 async-trait = "0.1.53"
 criterion = "0.4"
 csv = "1.1.6"

datafusion/core/src/physical_plan/file_format/csv.rs

@@ -448,7 +448,7 @@ mod tests {
         let err = it.next().await.unwrap().unwrap_err().to_string();
         assert_eq!(
             err,
-            "Csv error: incorrect number of fields, expected 14 got 13"
+            "Csv error: incorrect number of fields for line 1, expected 14 got 13"
         );
         Ok(())
     }

datafusion/core/tests/sql/timestamp.rs

@@ -1539,13 +1539,13 @@ async fn cast_timestamp_to_timestamptz() -> Result<()> {
     let actual = execute_to_batches(&ctx, sql).await;
 
     let expected = vec![
-        "+----------------------------------+---------------------------------------+",
-        "| table_a.ts                       | arrowtypeof(table_a.ts)               |",
-        "+----------------------------------+---------------------------------------+",
-        "| 2020-09-08T13:42:29.190855+00:00 | Timestamp(Nanosecond, Some(\"+00:00\")) |",
-        "| 2020-09-08T12:42:29.190855+00:00 | Timestamp(Nanosecond, Some(\"+00:00\")) |",
-        "| 2020-09-08T11:42:29.190855+00:00 | Timestamp(Nanosecond, Some(\"+00:00\")) |",
-        "+----------------------------------+---------------------------------------+",
+        "+-----------------------------+---------------------------------------+",
+        "| table_a.ts                  | arrowtypeof(table_a.ts)               |",
+        "+-----------------------------+---------------------------------------+",
+        "| 2020-09-08T13:42:29.190855Z | Timestamp(Nanosecond, Some(\"+00:00\")) |",
+        "| 2020-09-08T12:42:29.190855Z | Timestamp(Nanosecond, Some(\"+00:00\")) |",
+        "| 2020-09-08T11:42:29.190855Z | Timestamp(Nanosecond, Some(\"+00:00\")) |",
+        "+-----------------------------+---------------------------------------+",
     ];
     assert_batches_eq!(expected, &actual);
 

datafusion/expr/Cargo.toml

@@ -36,7 +36,7 @@ path = "src/lib.rs"
 
 [dependencies]
 ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
-arrow = { version = "30.0.1", default-features = false }
+arrow = { version = "31.0.0", default-features = false }
 datafusion-common = { path = "../common", version = "16.0.0" }
 log = "^0.4"
 sqlparser = "0.30"

datafusion/jit/Cargo.toml

@@ -36,7 +36,7 @@ path = "src/lib.rs"
 jit = []
 
 [dependencies]
-arrow = { version = "30.0.1", default-features = false }
+arrow = { version = "31.0.0", default-features = false }
 cranelift = "0.89.0"
 cranelift-jit = "0.89.0"
 cranelift-module = "0.89.0"

datafusion/optimizer/Cargo.toml

@@ -37,7 +37,7 @@ default = ["unicode_expressions"]
 unicode_expressions = []
 
 [dependencies]
-arrow = { version = "30.0.1", features = ["prettyprint"] }
+arrow = { version = "31.0.0", features = ["prettyprint"] }
 async-trait = "0.1.41"
 chrono = { version = "0.4.23", default-features = false }
 datafusion-common = { path = "../common", version = "16.0.0" }

datafusion/physical-expr/Cargo.toml

@@ -40,9 +40,9 @@ unicode_expressions = ["unicode-segmentation"]
 
 [dependencies]
 ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
-arrow = { version = "30.0.1", features = ["prettyprint", "dyn_cmp_dict"] }
-arrow-buffer = "30.0.1"
-arrow-schema = "30.0.1"
+arrow = { version = "31.0.0", features = ["prettyprint", "dyn_cmp_dict"] }
+arrow-buffer = "31.0.0"
+arrow-schema = "31.0.0"
 blake2 = { version = "^0.10.2", optional = true }
 blake3 = { version = "1.0", optional = true }
 chrono = { version = "0.4.23", default-features = false }

datafusion/proto/Cargo.toml

@@ -40,7 +40,7 @@ default = []
 json = ["pbjson", "serde", "serde_json"]
 
 [dependencies]
-arrow = "30.0.1"
+arrow = "31.0.0"
 chrono = { version = "0.4", default-features = false }
 datafusion = { path = "../core", version = "16.0.0" }
 datafusion-common = { path = "../common", version = "16.0.0" }
@@ -60,4 +60,4 @@ tokio = "1.18"
 [build-dependencies]
 # Pin these dependencies so that the generated output is deterministic
 pbjson-build = { version = "=0.5.1" }
-prost-build = { version = "=0.11.5" }
+prost-build = { version = "=0.11.6" }

datafusion/row/Cargo.toml

@@ -37,7 +37,7 @@ path = "src/lib.rs"
 jit = ["datafusion-jit"]
 
 [dependencies]
-arrow = "30.0.1"
+arrow = "31.0.0"
 datafusion-common = { path = "../common", version = "16.0.0" }
 datafusion-jit = { path = "../jit", version = "16.0.0", optional = true }
 paste = "^1.0"

datafusion/sql/Cargo.toml

@@ -37,7 +37,7 @@ default = ["unicode_expressions"]
 unicode_expressions = []
 
 [dependencies]
-arrow-schema = "30.0.1"
+arrow-schema = "31.0.0"
 datafusion-common = { path = "../common", version = "16.0.0" }
 datafusion-expr = { path = "../expr", version = "16.0.0" }
 log = "^0.4"

parquet-test-utils/Cargo.toml

@@ -25,4 +25,4 @@ edition = "2021"
 [dependencies]
 datafusion = { path = "../datafusion/core" }
 object_store = "0.5.0"
-parquet = "30.0.1"
+parquet = "31.0.0"

test-utils/Cargo.toml

@@ -23,7 +23,7 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-arrow = { version = "30.0.1", features = ["prettyprint"] }
+arrow = { version = "31.0.0", features = ["prettyprint"] }
 datafusion-common = { path = "../datafusion/common" }
 env_logger = "0.10.0"
 rand = "0.8"