Update to new arrow avro API (#4270)

Also reuse buffers in writing avro. This will improve performance when writing multiple chunks
pola-rs · Aug 5, 2022 · 833e588 · 833e588
1 parent 54746ad
commit 833e588
Show file tree

Hide file tree

Showing 8 changed files with 341 additions and 315 deletions.
diff --git a/polars/polars-arrow/Cargo.toml b/polars/polars-arrow/Cargo.toml
@@ -9,9 +9,9 @@ description = "Arrow interfaces for Polars DataFrame library"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-# arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "5711cfcbb785450697eb38e221b5dc1825550ee6", features = ["compute_concatenate"], default-features = false }
+arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "3b9d86bfb65bc786bf9315a41737a77895716443", features = ["compute_concatenate"], default-features = false }
 # arrow = { package = "arrow2", path = "../../../arrow2", features = ["compute_concatenate"], default-features = false }
-arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "ipc_meta", features = ["compute_concatenate"], default-features = false }
+# arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "ipc_meta", features = ["compute_concatenate"], default-features = false }
 # arrow = { package = "arrow2", version = "0.12", default-features = false, features = ["compute_concatenate"] }
 hashbrown = "0.12"
 num = "^0.4"

diff --git a/polars/polars-core/Cargo.toml b/polars/polars-core/Cargo.toml
@@ -173,10 +173,11 @@ thiserror = "^1.0"
 
 [dependencies.arrow]
 package = "arrow2"
-git = "https://github.com/ritchie46/arrow2"
-# rev = "5711cfcbb785450697eb38e221b5dc1825550ee6"
+git = "https://github.com/jorgecarleitao/arrow2"
+# git = "https://github.com/ritchie46/arrow2"
+rev = "3b9d86bfb65bc786bf9315a41737a77895716443"
 # path = "../../../arrow2"
-branch = "ipc_meta"
+# branch = "ipc_meta"
 # version = "0.12"
 default-features = false
 features = [

diff --git a/polars/polars-io/Cargo.toml b/polars/polars-io/Cargo.toml
@@ -37,8 +37,8 @@ private = ["polars-time/private"]
 [dependencies]
 ahash = "0.7"
 anyhow = "1.0"
-# arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "5711cfcbb785450697eb38e221b5dc1825550ee6", default-features = false }
-arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "ipc_meta", default-features = false }
+arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "3b9d86bfb65bc786bf9315a41737a77895716443", default-features = false }
+# arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "ipc_meta", default-features = false }
 # arrow = { package = "arrow2", version = "0.12", default-features = false }
 # arrow = { package = "arrow2", path = "../../../arrow2", default-features = false }
 csv-core = { version = "0.1.10", optional = true }

diff --git a/polars/polars-io/src/avro.rs b/polars/polars-io/src/avro.rs