upgrade arrow (#3316)

pola-rs · May 6, 2022 · ad2234e · ad2234e
1 parent 078fec7
commit ad2234e
Show file tree

Hide file tree

Showing 10 changed files with 26 additions and 15 deletions.
diff --git a/polars/polars-arrow/Cargo.toml b/polars/polars-arrow/Cargo.toml
@@ -9,7 +9,7 @@ description = "Arrow interfaces for Polars DataFrame library"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "3c64e7ab957880d9d6c72289c34fffd96a6c7030", default-features = false }
+arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "d0359641dbe6f9c6c50ab2f069b00a3b4d2cb5da", default-features = false }
 # arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "polars", default-features = false }
 # arrow = { package = "arrow2", version = "0.11", default-features = false, features = ["compute_concatenate"] }
 hashbrown = "0.12"

diff --git a/polars/polars-core/Cargo.toml b/polars/polars-core/Cargo.toml
@@ -171,7 +171,7 @@ thiserror = "^1.0"
 package = "arrow2"
 git = "https://github.com/jorgecarleitao/arrow2"
 # git = "https://github.com/ritchie46/arrow2"
-rev = "3c64e7ab957880d9d6c72289c34fffd96a6c7030"
+rev = "d0359641dbe6f9c6c50ab2f069b00a3b4d2cb5da"
 # branch = "polars"
 # version = "0.11"
 default-features = false

diff --git a/polars/polars-core/src/chunked_array/ops/aggregate.rs b/polars/polars-core/src/chunked_array/ops/aggregate.rs
@@ -766,22 +766,22 @@ impl ChunkAggSeries for Utf8Chunked {
 }
 
 macro_rules! one_null_list {
-    ($self:ident) => {{
-        let mut builder = get_list_builder(&DataType::Null, 0, 1, $self.name());
+    ($self:ident, $dtype: expr) => {{
+        let mut builder = get_list_builder(&$dtype, 0, 1, $self.name());
         builder.append_opt_series(None);
         builder.finish().into_series()
     }};
 }
 
 impl ChunkAggSeries for ListChunked {
     fn sum_as_series(&self) -> Series {
-        one_null_list!(self)
+        one_null_list!(self, self.inner_dtype())
     }
     fn max_as_series(&self) -> Series {
-        one_null_list!(self)
+        one_null_list!(self, self.inner_dtype())
     }
     fn min_as_series(&self) -> Series {
-        one_null_list!(self)
+        one_null_list!(self, self.inner_dtype())
     }
 }
 

diff --git a/polars/polars-core/src/series/mod.rs b/polars/polars-core/src/series/mod.rs
@@ -953,7 +953,7 @@ impl Series {
         let val = [self.mean()];
         let s = Series::new(self.name(), val);
         if !self.dtype().is_numeric() {
-            s.cast(self.dtype()).unwrap()
+            Series::full_null(self.name(), 1, self.dtype())
         } else {
             s
         }

diff --git a/polars/polars-io/Cargo.toml b/polars/polars-io/Cargo.toml
@@ -34,7 +34,7 @@ private = ["polars-time/private"]
 [dependencies]
 ahash = "0.7"
 anyhow = "1.0"
-arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "3c64e7ab957880d9d6c72289c34fffd96a6c7030", default-features = false }
+arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "d0359641dbe6f9c6c50ab2f069b00a3b4d2cb5da", default-features = false }
 # arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "polars", default-features = false }
 # arrow = { package = "arrow2", version = "0.11", default-features = false }
 csv-core = { version = "0.1.10", optional = true }

diff --git a/py-polars/Cargo.lock b/py-polars/Cargo.lock
diff --git a/py-polars/tests/conftest.py b/py-polars/tests/conftest.py
@@ -79,6 +79,9 @@ def df() -> pl.DataFrame:
             "date": [1324, 123, 1234],
             "datetime": [13241324, 12341256, 12341234],
             "time": [13241324, 12341256, 12341234],
+            "list_str": [["a", "b", None], ["a"], ["b"]],
+            "list_bool": [[True, False, None], [None], [True]],
+            "list_int": [[1, None, 3], [None], [1]],
         }
     )
     return df.with_columns(
@@ -91,6 +94,11 @@ def df() -> pl.DataFrame:
     )
 
 
+@pytest.fixture
+def df_no_lists(df: pl.DataFrame) -> pl.DataFrame:
+    return df.select(pl.all().exclude(["list_str", "list_int", "list_bool"]))
+
+
 @pytest.fixture
 def fruits_cars() -> pl.DataFrame:
     return pl.DataFrame(

diff --git a/py-polars/tests/io/test_csv.py b/py-polars/tests/io/test_csv.py
@@ -13,7 +13,8 @@
 from polars import DataType
 
 
-def test_to_from_buffer(df: pl.DataFrame) -> None:
+def test_to_from_buffer(df_no_lists: pl.DataFrame) -> None:
+    df = df_no_lists
     buf = io.BytesIO()
     df.write_csv(buf)
     buf.seek(0)
@@ -26,7 +27,8 @@ def test_to_from_buffer(df: pl.DataFrame) -> None:
     assert df.frame_equal(read_df)
 
 
-def test_to_from_file(io_test_dir: str, df: pl.DataFrame) -> None:
+def test_to_from_file(io_test_dir: str, df_no_lists: pl.DataFrame) -> None:
+    df = df_no_lists
     df = df.drop("strings_nulls")
 
     f = os.path.join(io_test_dir, "small.csv")

diff --git a/py-polars/tests/io/test_ipc.py b/py-polars/tests/io/test_ipc.py
@@ -85,8 +85,9 @@ def test_ipc_schema(compressions: List[str]) -> None:
 
 
 def test_ipc_schema_from_file(
-    io_test_dir: str, df: pl.DataFrame, compressions: List[str]
+    io_test_dir: str, df_no_lists: pl.DataFrame, compressions: List[str]
 ) -> None:
+    df = df_no_lists
     f_ipc = os.path.join(io_test_dir, "small.ipc")
 
     # does not yet work on windows because we hold an mmap?

diff --git a/py-polars/tests/io/test_parquet.py b/py-polars/tests/io/test_parquet.py
@@ -35,7 +35,7 @@ def test_to_from_buffer(df: pl.DataFrame, compressions: List[str]) -> None:
             df.write_parquet(buf, compression=compression)
             buf.seek(0)
             read_df = pl.read_parquet(buf)
-            assert df.frame_equal(read_df)
+            assert df.frame_equal(read_df, null_equal=True)
 
 
 def test_to_from_file(