update arrow

polars/polars-arrow/Cargo.toml

@@ -9,8 +9,8 @@ description = "Arrow interfaces for Polars DataFrame library"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-#arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "fa5d9eaead7eddce65624e67baabf75c975fb9bd", default-features = false }
-arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", default-features = false }
+arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "194a95d52978b60859a3b2846f8e86a89bc15990", default-features = false }
+#arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", default-features = false }
 #arrow = { package = "arrow2", version = "0.5.3", default-features=false}
 thiserror = "^1.0"
 num = "^0.4"

polars/polars-arrow/src/kernels/rolling/nulls.rs

@@ -1,6 +1,6 @@
 use super::*;
 use crate::utils::CustomIterTools;
-use arrow::array::{Array, ArrayRef, PrimitiveArray};
+use arrow::array::{ArrayRef, PrimitiveArray};
 use arrow::bitmap::utils::{count_zeros, get_bit_unchecked};
 use arrow::buffer::MutableBuffer;
 use arrow::types::NativeType;

polars/polars-arrow/src/kernels/set.rs

@@ -17,7 +17,7 @@ where
         return array.clone();
     }
 
-    let validity = array.validity().as_ref().unwrap();
+    let validity = array.validity().unwrap();
     let validity = BooleanArray::from_data_default(validity.clone(), None);
 
     let mut av = MutableBuffer::with_capacity(array.len());
@@ -90,7 +90,7 @@ where
     Ok(PrimitiveArray::from_data(
         data_type,
         buf.into(),
-        array.validity().clone(),
+        array.validity().cloned(),
     ))
 }
 

polars/polars-core/Cargo.toml

@@ -125,8 +125,8 @@ docs-selection = [
 ]
 
 [dependencies]
-#arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "fa5d9eaead7eddce65624e67baabf75c975fb9bd", default-features = false, features=["compute"] }
-arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", default-features = false, features=["compute"] }
+arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "194a95d52978b60859a3b2846f8e86a89bc15990", default-features = false, features=["compute"] }
+#arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", default-features = false, features=["compute"] }
 #arrow = { package = "arrow2", version="0.5.3", default-features = false, features=["compute"]}
 polars-arrow = {version = "0.16.0", path = "../polars-arrow"}
 thiserror = "1.0"

polars/polars-core/src/chunked_array/bitwise.rs

@@ -19,8 +19,7 @@ where
             .map(|(l_arr, r_arr)| {
                 let l_vals = l_arr.values().as_slice();
                 let r_vals = r_arr.values().as_slice();
-                let valididity =
-                    combine_validities(l_arr.validity().as_ref(), r_arr.validity().as_ref());
+                let valididity = combine_validities(l_arr.validity(), r_arr.validity());
 
                 let av = l_vals
                     .iter()
@@ -53,8 +52,7 @@ where
             .map(|(l_arr, r_arr)| {
                 let l_vals = l_arr.values().as_slice();
                 let r_vals = r_arr.values().as_slice();
-                let valididity =
-                    combine_validities(l_arr.validity().as_ref(), r_arr.validity().as_ref());
+                let valididity = combine_validities(l_arr.validity(), r_arr.validity());
 
                 let av = l_vals
                     .iter()
@@ -87,8 +85,7 @@ where
             .map(|(l_arr, r_arr)| {
                 let l_vals = l_arr.values().as_slice();
                 let r_vals = r_arr.values().as_slice();
-                let valididity =
-                    combine_validities(l_arr.validity().as_ref(), r_arr.validity().as_ref());
+                let valididity = combine_validities(l_arr.validity(), r_arr.validity());
 
                 let av = l_vals
                     .iter()
@@ -158,8 +155,7 @@ impl BitXor for &BooleanChunked {
             .downcast_iter()
             .zip(r.downcast_iter())
             .map(|(l_arr, r_arr)| {
-                let valididity =
-                    combine_validities(l_arr.validity().as_ref(), r_arr.validity().as_ref());
+                let valididity = combine_validities(l_arr.validity(), r_arr.validity());
 
                 let mut vals = MutableBitmap::with_capacity(l_arr.len());
 

polars/polars-core/src/chunked_array/cast.rs

@@ -3,7 +3,6 @@
 use crate::chunked_array::builder::CategoricalChunkedBuilder;
 use crate::chunked_array::kernels::cast_physical;
 use crate::prelude::*;
-use arrow::array::Array;
 use arrow::compute::cast;
 use num::NumCast;
 
@@ -246,7 +245,7 @@ fn cast_inner_list_type(
     let child = cast::cast(child.as_ref(), child_type)?.into();
 
     let data_type = ListArray::<i64>::default_datatype(child_type.clone());
-    let list = ListArray::from_data(data_type, offsets.clone(), child, list.validity().clone());
+    let list = ListArray::from_data(data_type, offsets.clone(), child, list.validity().cloned());
     Ok(Arc::new(list) as ArrayRef)
 }
 

polars/polars-core/src/chunked_array/kernels/mod.rs

@@ -8,7 +8,7 @@ pub(crate) mod take_agg;
 pub mod temporal;
 
 use crate::datatypes::{DataType, PolarsNumericType};
-use arrow::array::{Array, ArrayRef, BooleanArray, PrimitiveArray};
+use arrow::array::{ArrayRef, BooleanArray, PrimitiveArray};
 use arrow::bitmap::Bitmap;
 use arrow::types::NativeType;
 use num::Float;
@@ -38,7 +38,7 @@ where
 
     Arc::new(BooleanArray::from_data_default(
         values,
-        arr.validity().clone(),
+        arr.validity().cloned(),
     ))
 }
 
@@ -50,7 +50,7 @@ where
 
     Arc::new(BooleanArray::from_data_default(
         values,
-        arr.validity().clone(),
+        arr.validity().cloned(),
     ))
 }
 
@@ -62,7 +62,7 @@ where
 
     Arc::new(BooleanArray::from_data_default(
         values,
-        arr.validity().clone(),
+        arr.validity().cloned(),
     ))
 }
 
@@ -74,6 +74,6 @@ where
 
     Arc::new(BooleanArray::from_data_default(
         values,
-        arr.validity().clone(),
+        arr.validity().cloned(),
     ))
 }

polars/polars-core/src/chunked_array/kernels/strings.rs

@@ -1,5 +1,5 @@
 use crate::prelude::Arc;
-use arrow::array::{Array, ArrayRef, UInt32Array, Utf8Array};
+use arrow::array::{ArrayRef, UInt32Array, Utf8Array};
 use arrow::buffer::Buffer;
 use arrow::datatypes::DataType;
 
@@ -8,6 +8,6 @@ pub(crate) fn string_lengths(array: &Utf8Array<i64>) -> ArrayRef {
 
     let values = Buffer::from_trusted_len_iter(values);
 
-    let array = UInt32Array::from_data(DataType::UInt32, values, array.validity().clone());
+    let array = UInt32Array::from_data(DataType::UInt32, values, array.validity().cloned());
     Arc::new(array)
 }

polars/polars-core/src/chunked_array/kernels/take.rs

@@ -15,7 +15,7 @@ pub(crate) unsafe fn take_primitive_unchecked<T: PolarsNumericType>(
 ) -> Arc<PrimitiveArray<T::Native>> {
     let array_values = arr.values();
     let index_values = indices.values();
-    let validity_values = arr.validity().as_ref().expect("should have nulls");
+    let validity_values = arr.validity().expect("should have nulls");
 
     // first take the values, these are always needed
     let values: AlignedVec<T::Native> = index_values
@@ -72,7 +72,7 @@ pub(crate) unsafe fn take_no_null_primitive<T: PolarsPrimitiveType>(
         .map(|idx| *array_values.get_unchecked(*idx as usize));
 
     let values = Buffer::from_trusted_len_iter(iter);
-    let validity = indices.validity().clone();
+    let validity = indices.validity().cloned();
     Arc::new(PrimitiveArray::from_data(
         T::get_dtype().to_arrow(),
         values,
@@ -121,7 +121,7 @@ pub(crate) unsafe fn take_primitive_iter_unchecked<
     indices: I,
 ) -> Arc<PrimitiveArray<T::Native>> {
     let array_values = arr.values().as_slice();
-    let validity = arr.validity().as_ref().expect("should have nulls");
+    let validity = arr.validity().expect("should have nulls");
 
     let iter = indices.into_iter().map(|idx| {
         if validity.get_bit_unchecked(idx) {
@@ -172,7 +172,7 @@ pub(crate) unsafe fn take_primitive_opt_iter_unchecked<
     indices: I,
 ) -> Arc<PrimitiveArray<T::Native>> {
     let array_values = arr.values().as_slice();
-    let validity = arr.validity().as_ref().expect("should have nulls");
+    let validity = arr.validity().expect("should have nulls");
 
     let iter = indices.into_iter().map(|opt_idx| {
         opt_idx.and_then(|idx| {
@@ -238,7 +238,7 @@ pub(crate) unsafe fn take_bool_iter_unchecked<I: IntoIterator<Item = usize>>(
     arr: &BooleanArray,
     indices: I,
 ) -> Arc<BooleanArray> {
-    let validity = arr.validity().as_ref().expect("should have nulls");
+    let validity = arr.validity().expect("should have nulls");
 
     let iter = indices.into_iter().map(|idx| {
         if validity.get_bit_unchecked(idx) {
@@ -259,7 +259,7 @@ pub(crate) unsafe fn take_bool_opt_iter_unchecked<I: IntoIterator<Item = Option<
     arr: &BooleanArray,
     indices: I,
 ) -> Arc<BooleanArray> {
-    let validity = arr.validity().as_ref().expect("should have nulls");
+    let validity = arr.validity().expect("should have nulls");
     let iter = indices.into_iter().map(|opt_idx| {
         opt_idx.and_then(|idx| {
             if validity.get_bit_unchecked(idx) {
@@ -309,7 +309,7 @@ pub(crate) unsafe fn take_utf8_iter_unchecked<I: IntoIterator<Item = usize>>(
     arr: &LargeStringArray,
     indices: I,
 ) -> Arc<LargeStringArray> {
-    let validity = arr.validity().as_ref().expect("should have nulls");
+    let validity = arr.validity().expect("should have nulls");
     let iter = indices.into_iter().map(|idx| {
         if validity.get_bit_unchecked(idx) {
             Some(arr.value_unchecked(idx))
@@ -342,7 +342,7 @@ pub(crate) unsafe fn take_utf8_opt_iter_unchecked<I: IntoIterator<Item = Option<
     arr: &LargeStringArray,
     indices: I,
 ) -> Arc<LargeStringArray> {
-    let validity = arr.validity().as_ref().expect("should have nulls");
+    let validity = arr.validity().expect("should have nulls");
     let iter = indices.into_iter().map(|opt_idx| {
         opt_idx.and_then(|idx| {
             if validity.get_bit_unchecked(idx) {
@@ -421,10 +421,10 @@ pub(crate) unsafe fn take_utf8_unchecked(
                 }
                 *offset = length_so_far;
             });
-        validity = indices.validity().clone();
+        validity = indices.validity().cloned();
     } else {
         let mut builder = MutableUtf8Array::with_capacities(data_len, length_so_far as usize);
-        let validity_arr = arr.validity().as_ref().expect("should have nulls");
+        let validity_arr = arr.validity().expect("should have nulls");
 
         if indices.null_count() == 0 {
             (0..data_len).for_each(|idx| {
@@ -437,7 +437,7 @@ pub(crate) unsafe fn take_utf8_unchecked(
                 });
             });
         } else {
-            let validity_indices = indices.validity().as_ref().expect("should have nulls");
+            let validity_indices = indices.validity().expect("should have nulls");
             (0..data_len).for_each(|idx| {
                 if validity_indices.get_bit_unchecked(idx) {
                     let index = indices.value_unchecked(idx) as usize;
@@ -510,7 +510,7 @@ unsafe fn take_value_indices_from_list(
             }
         }
     } else {
-        let validity = indices.validity().as_ref().expect("should have nulls");
+        let validity = indices.validity().expect("should have nulls");
 
         for i in 0..indices.len() {
             if validity.get_bit_unchecked(i) {

polars/polars-core/src/chunked_array/kernels/take_agg.rs

@@ -39,10 +39,7 @@ pub(crate) unsafe fn take_agg_primitive_iter_unchecked<
     }
 
     let array_values = arr.values().as_slice();
-    let validity = arr
-        .validity()
-        .as_ref()
-        .expect("null buffer should be there");
+    let validity = arr.validity().expect("null buffer should be there");
 
     let out = indices.into_iter().fold(init, |acc, idx| {
         if validity.get_bit_unchecked(idx) {
@@ -74,10 +71,7 @@ pub(crate) unsafe fn take_agg_primitive_iter_unchecked_count_nulls<
     }
 
     let array_values = arr.values().as_slice();
-    let validity = arr
-        .validity()
-        .as_ref()
-        .expect("null buffer should be there");
+    let validity = arr.validity().expect("null buffer should be there");
 
     let mut null_count = 0;
     let out = indices.into_iter().fold(init, |acc, idx| {

polars/polars-core/src/chunked_array/mod.rs

@@ -199,7 +199,7 @@ impl<T> ChunkedArray<T> {
     }
 
     /// Get the buffer of bits representing null values
-    pub fn null_bits(&self) -> impl Iterator<Item = (usize, &Option<Bitmap>)> + '_ {
+    pub fn null_bits(&self) -> impl Iterator<Item = (usize, Option<&Bitmap>)> + '_ {
         self.chunks
             .iter()
             .map(|arr| (arr.null_count(), arr.validity()))
@@ -389,7 +389,6 @@ impl<T> ChunkedArray<T> {
             .map(|arr| {
                 let bitmap = arr
                     .validity()
-                    .as_ref()
                     .map(|bitmap| !bitmap)
                     .unwrap_or_else(|| Bitmap::new_zeroed(arr.len()));
                 Arc::new(BooleanArray::from_data_default(bitmap, None)) as ArrayRef
@@ -409,7 +408,7 @@ impl<T> ChunkedArray<T> {
             .map(|arr| {
                 let bitmap = arr
                     .validity()
-                    .clone()
+                    .cloned()
                     .unwrap_or_else(|| !(&Bitmap::new_zeroed(arr.len())));
                 Arc::new(BooleanArray::from_data_default(bitmap, None)) as ArrayRef
             })

polars/polars-core/src/chunked_array/object/mod.rs

@@ -122,8 +122,8 @@ where
         self.len
     }
 
-    fn validity(&self) -> &Option<Bitmap> {
-        &self.null_bitmap
+    fn validity(&self) -> Option<&Bitmap> {
+        self.null_bitmap.as_ref()
     }
     fn with_validity(&self, validity: Option<Bitmap>) -> Box<dyn Array> {
         let mut arr = self.clone();

polars/polars-core/src/chunked_array/ops/apply.rs

@@ -63,7 +63,7 @@ where
             .zip(self.null_bits())
             .map(|(slice, (_, validity))| {
                 let values = AlignedVec::<_>::from_trusted_len_iter(slice.iter().map(|&v| f(v)));
-                to_array::<S>(values, validity.clone())
+                to_array::<S>(values, validity.cloned())
             })
             .collect();
         ChunkedArray::<S>::new_from_chunks(self.name(), chunks)
@@ -101,7 +101,7 @@ where
             .map(|(slice, (_, validity))| {
                 let values = slice.iter().copied().map(f);
                 let values = AlignedVec::<_>::from_trusted_len_iter(values);
-                to_array::<T>(values, validity.clone())
+                to_array::<T>(values, validity.cloned())
             })
             .collect();
         ChunkedArray::<T>::new_from_chunks(self.name(), chunks)
@@ -117,7 +117,7 @@ where
             .zip(self.null_bits())
             .map(|(slice, (_null_count, validity))| {
                 let vec: Result<AlignedVec<_>> = slice.iter().copied().map(f).collect();
-                Ok((vec?, validity.clone()))
+                Ok((vec?, validity.cloned()))
             })
             .collect::<Result<_>>()?;
         ca.rename(self.name());
@@ -200,7 +200,7 @@ impl<'a> ChunkApply<'a, bool, bool> for BooleanChunked {
         self.apply_kernel_cast(|array| {
             let values = array.values().iter().map(f);
             let values = AlignedVec::<_>::from_trusted_len_iter(values);
-            let validity = array.validity().clone();
+            let validity = array.validity().cloned();
             to_array::<S>(values, validity)
         })
     }
@@ -282,7 +282,7 @@ impl<'a> ChunkApply<'a, &'a str, Cow<'a, str>> for Utf8Chunked {
             .map(|array| {
                 let values = array.values_iter().map(|x| f(x));
                 let values = AlignedVec::<_>::from_trusted_len_iter(values);
-                to_array::<S>(values, array.validity().clone())
+                to_array::<S>(values, array.validity().cloned())
             })
             .collect();
         ChunkedArray::new_from_chunks(self.name(), chunks)
@@ -299,7 +299,7 @@ impl<'a> ChunkApply<'a, &'a str, Cow<'a, str>> for Utf8Chunked {
             .map(|array| {
                 let values = array.into_iter().map(|x| f(x));
                 let values = AlignedVec::<_>::from_trusted_len_iter(values);
-                to_array::<S>(values, array.validity().clone())
+                to_array::<S>(values, array.validity().cloned())
             })
             .collect();
         ChunkedArray::new_from_chunks(self.name(), chunks)
@@ -445,7 +445,7 @@ impl<'a> ChunkApply<'a, Series, Series> for ListChunked {
                         f(series)
                     })
                     .collect_trusted();
-                to_array::<S>(values, array.validity().clone())
+                to_array::<S>(values, array.validity().cloned())
             })
             .collect();
         ChunkedArray::new_from_chunks(self.name(), chunks)
@@ -468,7 +468,7 @@ impl<'a> ChunkApply<'a, Series, Series> for ListChunked {
                     f(x)
                 });
                 let values = AlignedVec::<_>::from_trusted_len_iter(values);
-                to_array::<S>(values, array.validity().clone())
+                to_array::<S>(values, array.validity().cloned())
             })
             .collect();
         ChunkedArray::new_from_chunks(self.name(), chunks)