upgrade arrow

polars/polars-arrow/Cargo.toml

@@ -9,7 +9,7 @@ description = "Arrow interfaces for Polars DataFrame library"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "d0359641dbe6f9c6c50ab2f069b00a3b4d2cb5da", features = ["compute_concatenate"], default-features = false }
+arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "b9aa8e8da7648559efdc78535085a509a281c55d", features = ["compute_concatenate"], default-features = false }
 # arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "polars", default-features = false }
 # arrow = { package = "arrow2", version = "0.11", default-features = false, features = ["compute_concatenate"] }
 hashbrown = "0.12"

polars/polars-core/Cargo.toml

@@ -171,7 +171,7 @@ thiserror = "^1.0"
 package = "arrow2"
 git = "https://github.com/jorgecarleitao/arrow2"
 # git = "https://github.com/ritchie46/arrow2"
-rev = "d0359641dbe6f9c6c50ab2f069b00a3b4d2cb5da"
+rev = "b9aa8e8da7648559efdc78535085a509a281c55d"
 # branch = "polars"
 # version = "0.11"
 default-features = false

polars/polars-io/Cargo.toml

@@ -34,7 +34,7 @@ private = ["polars-time/private"]
 [dependencies]
 ahash = "0.7"
 anyhow = "1.0"
-arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "d0359641dbe6f9c6c50ab2f069b00a3b4d2cb5da", default-features = false }
+arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "b9aa8e8da7648559efdc78535085a509a281c55d", default-features = false }
 # arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "polars", default-features = false }
 # arrow = { package = "arrow2", version = "0.11", default-features = false }
 csv-core = { version = "0.1.10", optional = true }

polars/polars-io/src/parquet/write.rs

@@ -110,15 +110,11 @@ where
                 let columns = batch
                     .columns()
                     .par_iter()
-                    .zip(parquet_schema.columns().par_iter())
+                    .zip(parquet_schema.fields().par_iter())
                     .zip(encodings.par_iter())
-                    .map(|((array, descriptor), encoding)| {
-                        let encoded_pages = array_to_pages(
-                            array.as_ref(),
-                            descriptor.descriptor.clone(),
-                            options,
-                            *encoding,
-                        )?;
+                    .map(|((array, tp), encoding)| {
+                        let encoded_pages =
+                            array_to_pages(array.as_ref(), tp.clone(), options, *encoding)?;
                         encoded_pages
                             .map(|page| {
                                 compress(page?, vec![], options.compression).map_err(|x| x.into())

py-polars/tests/conftest.py

@@ -79,9 +79,10 @@ def df() -> pl.DataFrame:
             "date": [1324, 123, 1234],
             "datetime": [13241324, 12341256, 12341234],
             "time": [13241324, 12341256, 12341234],
-            "list_str": [["a", "b", None], ["a"], ["b"]],
-            "list_bool": [[True, False, None], [None], [True]],
-            "list_int": [[1, None, 3], [None], [1]],
+            "list_str": [["a", "b", None], ["a"], []],
+            "list_bool": [[True, False, None], [None], []],
+            "list_int": [[1, None, 3], [None], []],
+            "list_flt": [[1.0, None, 3.0], [None], []],
         }
     )
     return df.with_columns(
@@ -96,7 +97,9 @@ def df() -> pl.DataFrame:
 
 @pytest.fixture
 def df_no_lists(df: pl.DataFrame) -> pl.DataFrame:
-    return df.select(pl.all().exclude(["list_str", "list_int", "list_bool"]))
+    return df.select(
+        pl.all().exclude(["list_str", "list_int", "list_bool", "list_int", "list_flt"])
+    )
 
 
 @pytest.fixture