update arrow (#3762)

polars/polars-arrow/Cargo.toml

@@ -9,7 +9,7 @@ description = "Arrow interfaces for Polars DataFrame library"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "d1ab4efa5c895513681bacd5e20660be882433ac", features = ["compute_concatenate"], default-features = false }
+arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "e3a005dfb0ff122eff7041a75be6d50b76acd032", features = ["compute_concatenate"], default-features = false }
 # arrow = { package = "arrow2", path = "../../../arrow2", features = ["compute_concatenate"], default-features = false }
 # arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "arity_assign", features = ["compute_concatenate"], default-features = false }
 # arrow = { package = "arrow2", version = "0.12", default-features = false, features = ["compute_concatenate"] }

polars/polars-core/Cargo.toml

@@ -174,7 +174,7 @@ thiserror = "^1.0"
 package = "arrow2"
 git = "https://github.com/jorgecarleitao/arrow2"
 # git = "https://github.com/ritchie46/arrow2"
-rev = "d1ab4efa5c895513681bacd5e20660be882433ac"
+rev = "e3a005dfb0ff122eff7041a75be6d50b76acd032"
 # path = "../../../arrow2"
 # branch = "arity_assign"
 # version = "0.12"

polars/polars-core/src/chunked_array/object/extension/mod.rs

@@ -69,6 +69,7 @@ pub(crate) fn create_extension<
     let n_padding = (buf.as_ptr() as usize) % t_alignment;
     buf.extend(std::iter::repeat(0).take(n_padding));
 
+    let mut null_count = 0 as IdxSize;
     // transmute T as bytes and copy in buffer
     for opt_t in iter.into_iter() {
         match opt_t {
@@ -81,6 +82,7 @@ pub(crate) fn create_extension<
                 mem::forget(t);
             }
             None => {
+                null_count += 1;
                 unsafe {
                     buf.extend_from_slice(any_as_u8_slice(&T::default()));
                     // Safety: we allocated upfront
@@ -117,7 +119,8 @@ pub(crate) fn create_extension<
         physical_type.into(),
         Some(metadata),
     );
-    let validity = if validity.null_count() > 0 {
+    // first freeze, otherwise we compute null
+    let validity = if null_count > 0 {
         Some(validity.into())
     } else {
         None

polars/polars-core/src/chunked_array/ops/aggregate.rs

@@ -462,9 +462,9 @@ impl ChunkAgg<IdxSize> for BooleanChunked {
                 self.downcast_iter()
                     .map(|arr| match arr.validity() {
                         Some(validity) => {
-                            (arr.len() - (validity & arr.values()).null_count()) as IdxSize
+                            (arr.len() - (validity & arr.values()).unset_bits()) as IdxSize
                         }
-                        None => (arr.len() - arr.values().null_count()) as IdxSize,
+                        None => (arr.len() - arr.values().unset_bits()) as IdxSize,
                     })
                     .sum(),
             )

polars/polars-core/src/chunked_array/ops/unique/rank.rs

@@ -205,7 +205,7 @@ pub(crate) fn rank(s: &Series, method: RankMethod, reverse: bool) -> Series {
             }
 
             let bitmap = obs.values();
-            let cap = bitmap.len() - bitmap.null_count();
+            let cap = bitmap.len() - bitmap.unset_bits();
             let mut count = Vec::with_capacity(cap + 1);
             let mut cnt: IdxSize = 0;
             count.push(cnt);

polars/polars-core/src/chunked_array/ops/zip.rs

@@ -17,7 +17,7 @@ fn prepare_mask(mask: &BooleanArray) -> BooleanArray {
 
     match mask.validity() {
         // nulls are set to true meaning we take from the right in the zip/ if_then_else kernel
-        Some(validity) if validity.null_count() != 0 => {
+        Some(validity) if validity.unset_bits() != 0 => {
             let mask = mask.values() & validity;
             BooleanArray::from_data_default(mask, None)
         }

polars/polars-core/src/frame/row.rs

@@ -522,7 +522,7 @@ where
 
                 let validity = if has_nulls {
                     let validity = Bitmap::from_trusted_len_iter(validity.iter().copied());
-                    if validity.null_count() > 0 {
+                    if validity.unset_bits() > 0 {
                         Some(validity)
                     } else {
                         None

polars/polars-core/src/series/implementations/struct_.rs

@@ -273,7 +273,7 @@ impl SeriesTrait for SeriesWrap<StructChunked> {
                         _ => {}
                     }
                     if let Some(validity) = &validity_agg {
-                        null_count += validity.null_count()
+                        null_count += validity.unset_bits()
                     }
                 }
             }

polars/polars-io/Cargo.toml

@@ -35,7 +35,7 @@ private = ["polars-time/private"]
 [dependencies]
 ahash = "0.7"
 anyhow = "1.0"
-arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "d1ab4efa5c895513681bacd5e20660be882433ac", default-features = false }
+arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "e3a005dfb0ff122eff7041a75be6d50b76acd032", default-features = false }
 # arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "arity_assign", default-features = false }
 # arrow = { package = "arrow2", version = "0.12", default-features = false }
 # arrow = { package = "arrow2", path = "../../../arrow2", default-features = false }