diff --git a/examples/python_rust_compiled_function/src/ffi.rs b/examples/python_rust_compiled_function/src/ffi.rs index b02431e85578..4c4fe66e378b 100644 --- a/examples/python_rust_compiled_function/src/ffi.rs +++ b/examples/python_rust_compiled_function/src/ffi.rs @@ -1,4 +1,4 @@ -use arrow::{array::ArrayRef, ffi}; +use arrow::ffi; use polars::prelude::*; use polars_arrow::export::arrow; use pyo3::exceptions::PyValueError; @@ -24,8 +24,7 @@ fn array_to_rust(arrow_array: &PyAny) -> PyResult { unsafe { let field = ffi::import_field_from_c(schema.as_ref()).unwrap(); - let array = ffi::import_array_from_c(array, field.data_type).unwrap(); - Ok(array.into()) + Ok(ffi::import_array_from_c(array, field.data_type).unwrap()) } } diff --git a/polars/polars-arrow/Cargo.toml b/polars/polars-arrow/Cargo.toml index 8784e77e1f03..d294e8f22c47 100644 --- a/polars/polars-arrow/Cargo.toml +++ b/polars/polars-arrow/Cargo.toml @@ -9,9 +9,9 @@ description = "Arrow interfaces for Polars DataFrame library" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "6919cc3281c3ee7dd3e973a7b303c67277458f20", features = ["compute_concatenate"], default-features = false } +arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "39db6fb7514364bfea08d594793b23e1ed5a7def", features = ["compute_concatenate"], default-features = false } # arrow = { package = "arrow2", path = "../../../arrow2", features = ["compute_concatenate"], default-features = false } -# arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "improve_mutable", features = ["compute_concatenate"], default-features = false } +# arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "count_shared", features = ["compute_concatenate"], default-features = false } # arrow = { package = "arrow2", version = "0.12", default-features = false, features = ["compute_concatenate"] } hashbrown = "0.12" num = "^0.4" diff --git a/polars/polars-arrow/src/array/list.rs b/polars/polars-arrow/src/array/list.rs index ba57bcee39ff..7ad5c81e4805 100644 --- a/polars/polars-arrow/src/array/list.rs +++ b/polars/polars-arrow/src/array/list.rs @@ -1,4 +1,5 @@ -use arrow::array::{Array, ArrayRef, ListArray}; +use crate::prelude::*; +use arrow::array::{Array, ListArray}; use arrow::bitmap::MutableBitmap; use arrow::compute::concatenate; use arrow::datatypes::DataType; @@ -87,11 +88,15 @@ impl<'a> AnonymousBuilder<'a> { let values = concatenate::concatenate(&self.arrays)?; let dtype = ListArray::::default_datatype(inner_dtype.clone()); - Ok(ListArray::::from_data( - dtype, - self.offsets.into(), - values.into(), - self.validity.map(|validity| validity.into()), - )) + // Safety: + // offsets are monotonically increasing + unsafe { + Ok(ListArray::::new_unchecked( + dtype, + self.offsets.into(), + values, + self.validity.map(|validity| validity.into()), + )) + } } } diff --git a/polars/polars-arrow/src/array/mod.rs b/polars/polars-arrow/src/array/mod.rs index 410caf66634d..3bfee29d35d9 100644 --- a/polars/polars-arrow/src/array/mod.rs +++ b/polars/polars-arrow/src/array/mod.rs @@ -1,8 +1,8 @@ -use arrow::array::{Array, ArrayRef, BooleanArray, ListArray, PrimitiveArray, Utf8Array}; +use crate::prelude::*; +use arrow::array::{Array, BooleanArray, ListArray, PrimitiveArray, Utf8Array}; use arrow::bitmap::MutableBitmap; use arrow::datatypes::DataType; use arrow::types::NativeType; -use std::sync::Arc; use crate::utils::CustomIterTools; @@ -94,10 +94,12 @@ pub trait ListFromIter { let values: PrimitiveArray = iter_to_values!(iterator, validity, offsets, length_so_far); - ListArray::from_data( + // Safety: + // offsets are monotonically increasing + ListArray::new_unchecked( ListArray::::default_datatype(data_type.clone()), offsets.into(), - Arc::new(values.to(data_type)), + Box::new(values.to(data_type)), Some(validity.into()), ) } @@ -122,10 +124,12 @@ pub trait ListFromIter { let values: BooleanArray = iter_to_values!(iterator, validity, offsets, length_so_far); - ListArray::from_data( + // Safety: + // Offsets are monotonically increasing. + ListArray::new_unchecked( ListArray::::default_datatype(DataType::Boolean), offsets.into(), - Arc::new(values), + Box::new(values), Some(validity.into()), ) } @@ -166,10 +170,12 @@ pub trait ListFromIter { .trust_my_length(n_elements) .collect(); - ListArray::from_data( + // Safety: + // offsets are monotonically increasing + ListArray::new_unchecked( ListArray::::default_datatype(DataType::LargeUtf8), offsets.into(), - Arc::new(values), + Box::new(values), Some(validity.into()), ) } diff --git a/polars/polars-arrow/src/conversion.rs b/polars/polars-arrow/src/conversion.rs index 201fb63ba665..ece200cfbc31 100644 --- a/polars/polars-arrow/src/conversion.rs +++ b/polars/polars-arrow/src/conversion.rs @@ -1,4 +1,5 @@ -use arrow::array::{ArrayRef, StructArray}; +use crate::prelude::*; +use arrow::array::StructArray; use arrow::chunk::Chunk; use arrow::datatypes::{DataType, Field}; diff --git a/polars/polars-arrow/src/data_types.rs b/polars/polars-arrow/src/data_types.rs index 933a4ea0a556..b5b43c79a4ec 100644 --- a/polars/polars-arrow/src/data_types.rs +++ b/polars/polars-arrow/src/data_types.rs @@ -1,3 +1,5 @@ +use arrow::array::Array; + /// # Safety /// unsafe code downstream relies on the correct is_float call pub unsafe trait IsFloat: private::Sealed { @@ -57,3 +59,5 @@ macro_rules! impl_is_float { impl_is_float!(f32); impl_is_float!(f64); + +pub type ArrayRef = Box; diff --git a/polars/polars-arrow/src/kernels/concatenate.rs b/polars/polars-arrow/src/kernels/concatenate.rs index 239d44c5a7db..7f2c871eb3f1 100644 --- a/polars/polars-arrow/src/kernels/concatenate.rs +++ b/polars/polars-arrow/src/kernels/concatenate.rs @@ -1,11 +1,10 @@ +use crate::prelude::*; use arrow::array::growable::make_growable; -use arrow::array::{Array, ArrayRef}; use arrow::error::{Error as ArrowError, Result}; -use std::sync::Arc; /// Concatenate multiple [Array] of the same type into a single [`Array`]. /// This does not check the arrays types. -pub fn concatenate_owned_unchecked(arrays: &[ArrayRef]) -> Result> { +pub fn concatenate_owned_unchecked(arrays: &[ArrayRef]) -> Result { if arrays.is_empty() { return Err(ArrowError::InvalidArgumentError( "concat requires input of at least one array".to_string(), @@ -26,5 +25,5 @@ pub fn concatenate_owned_unchecked(arrays: &[ArrayRef]) -> Result mutable.extend(i, 0, *len) } - Ok(mutable.as_arc()) + Ok(mutable.as_box()) } diff --git a/polars/polars-arrow/src/kernels/float.rs b/polars/polars-arrow/src/kernels/float.rs index 591c092a33ad..797781258c7f 100644 --- a/polars/polars-arrow/src/kernels/float.rs +++ b/polars/polars-arrow/src/kernels/float.rs @@ -1,9 +1,9 @@ use crate::array::default_arrays::FromData; -use arrow::array::{ArrayRef, BooleanArray, PrimitiveArray}; +use crate::prelude::*; +use arrow::array::{BooleanArray, PrimitiveArray}; use arrow::bitmap::Bitmap; use arrow::types::NativeType; use num::Float; -use std::sync::Arc; pub fn is_nan(arr: &PrimitiveArray) -> ArrayRef where @@ -11,7 +11,7 @@ where { let values = Bitmap::from_trusted_len_iter(arr.values().iter().map(|v| v.is_nan())); - Arc::new(BooleanArray::from_data_default( + Box::new(BooleanArray::from_data_default( values, arr.validity().cloned(), )) @@ -23,7 +23,7 @@ where { let values = Bitmap::from_trusted_len_iter(arr.values().iter().map(|v| !v.is_nan())); - Arc::new(BooleanArray::from_data_default( + Box::new(BooleanArray::from_data_default( values, arr.validity().cloned(), )) @@ -35,7 +35,7 @@ where { let values = Bitmap::from_trusted_len_iter(arr.values().iter().map(|v| v.is_finite())); - Arc::new(BooleanArray::from_data_default( + Box::new(BooleanArray::from_data_default( values, arr.validity().cloned(), )) @@ -47,7 +47,7 @@ where { let values = Bitmap::from_trusted_len_iter(arr.values().iter().map(|v| v.is_infinite())); - Arc::new(BooleanArray::from_data_default( + Box::new(BooleanArray::from_data_default( values, arr.validity().cloned(), )) diff --git a/polars/polars-arrow/src/kernels/list.rs b/polars/polars-arrow/src/kernels/list.rs index 12c4972dbddd..014ce2c273ce 100644 --- a/polars/polars-arrow/src/kernels/list.rs +++ b/polars/polars-arrow/src/kernels/list.rs @@ -1,8 +1,8 @@ -use crate::index::*; use crate::kernels::take::take_unchecked; +use crate::prelude::*; use crate::trusted_len::PushUnchecked; use crate::utils::CustomIterTools; -use arrow::array::{ArrayRef, ListArray}; +use arrow::array::ListArray; use arrow::buffer::Buffer; /// Get the indices that would result in a get operation on the lists values. @@ -81,7 +81,9 @@ pub fn array_to_unit_list(array: ArrayRef) -> ListArray { let offsets: Buffer = offsets.into(); let dtype = ListArray::::default_datatype(array.data_type().clone()); - ListArray::::from_data(dtype, offsets, array, None) + // Safety: + // offsets are monotonically increasing + unsafe { ListArray::::new_unchecked(dtype, offsets, array, None) } } #[cfg(test)] @@ -97,7 +99,7 @@ mod test { let offsets = Buffer::from(vec![0i64, 3, 5, 6]); let dtype = ListArray::::default_datatype(DataType::Int32); - ListArray::::from_data(dtype, offsets, Arc::new(values), None) + ListArray::::from_data(dtype, offsets, Box::new(values), None) } #[test] diff --git a/polars/polars-arrow/src/kernels/rolling/mod.rs b/polars/polars-arrow/src/kernels/rolling/mod.rs index 80c04585c4ae..141e25d53ffa 100644 --- a/polars/polars-arrow/src/kernels/rolling/mod.rs +++ b/polars/polars-arrow/src/kernels/rolling/mod.rs @@ -3,16 +3,15 @@ pub mod nulls; mod window; use crate::data_types::IsFloat; -use crate::prelude::QuantileInterpolOptions; +use crate::prelude::*; use crate::utils::CustomIterTools; -use arrow::array::{ArrayRef, PrimitiveArray}; +use arrow::array::PrimitiveArray; use arrow::bitmap::{Bitmap, MutableBitmap}; use arrow::types::NativeType; use num::ToPrimitive; use num::{Bounded, Float, NumCast, One, Zero}; use std::cmp::Ordering; use std::ops::{Add, AddAssign, Div, Mul, Sub, SubAssign}; -use std::sync::Arc; use window::*; type Start = usize; diff --git a/polars/polars-arrow/src/kernels/rolling/no_nulls/mod.rs b/polars/polars-arrow/src/kernels/rolling/no_nulls/mod.rs index 53591d7a4800..60903ce17e27 100644 --- a/polars/polars-arrow/src/kernels/rolling/no_nulls/mod.rs +++ b/polars/polars-arrow/src/kernels/rolling/no_nulls/mod.rs @@ -6,14 +6,13 @@ mod variance; use super::*; use crate::utils::CustomIterTools; -use arrow::array::{ArrayRef, PrimitiveArray}; +use arrow::array::PrimitiveArray; use arrow::datatypes::DataType; use arrow::types::NativeType; use num::{Float, NumCast}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use std::fmt::Debug; -use std::sync::Arc; pub use mean::*; pub use min_max::*; @@ -56,7 +55,7 @@ where .collect_trusted::>(); let validity = create_validity(min_periods, len as usize, window_size, det_offsets_fn); - Arc::new(PrimitiveArray::from_data( + Box::new(PrimitiveArray::from_data( T::PRIMITIVE.into(), out.into(), validity.map(|b| b.into()), @@ -104,7 +103,7 @@ where .collect_trusted::>(); let validity = create_validity(min_periods, len as usize, window_size, det_offsets_fn); - Arc::new(PrimitiveArray::from_data( + Box::new(PrimitiveArray::from_data( DataType::from(T::PRIMITIVE), out.into(), validity.map(|b| b.into()), diff --git a/polars/polars-arrow/src/kernels/rolling/no_nulls/quantile.rs b/polars/polars-arrow/src/kernels/rolling/no_nulls/quantile.rs index d0735f19fcd5..9f7b0c8f4ae3 100644 --- a/polars/polars-arrow/src/kernels/rolling/no_nulls/quantile.rs +++ b/polars/polars-arrow/src/kernels/rolling/no_nulls/quantile.rs @@ -26,7 +26,7 @@ where { if values.is_empty() { let out: Vec = vec![]; - return Arc::new(PrimitiveArray::from_data( + return Box::new(PrimitiveArray::from_data( T::PRIMITIVE.into(), out.into(), None, @@ -50,7 +50,7 @@ where }) .collect::>(); - Arc::new(out) + Box::new(out) } pub(crate) fn compute_quantile2( @@ -244,7 +244,7 @@ where .collect_trusted::>(); let validity = create_validity(min_periods, len as usize, window_size, det_offsets_fn); - Arc::new(PrimitiveArray::from_data( + Box::new(PrimitiveArray::from_data( T::PRIMITIVE.into(), out.into(), validity.map(|b| b.into()), @@ -284,7 +284,7 @@ where .collect_trusted::>(); let validity = create_validity(min_periods, len as usize, window_size, det_offsets_fn); - Arc::new(PrimitiveArray::from_data( + Box::new(PrimitiveArray::from_data( T::PRIMITIVE.into(), out.into(), validity.map(|b| b.into()), diff --git a/polars/polars-arrow/src/kernels/rolling/nulls/mod.rs b/polars/polars-arrow/src/kernels/rolling/nulls/mod.rs index ef8308bbaad2..d8a5bc0d0b75 100644 --- a/polars/polars-arrow/src/kernels/rolling/nulls/mod.rs +++ b/polars/polars-arrow/src/kernels/rolling/nulls/mod.rs @@ -76,7 +76,7 @@ where }) .collect_trusted::>(); - Arc::new(PrimitiveArray::from_data( + Box::new(PrimitiveArray::from_data( T::PRIMITIVE.into(), out.into(), Some(validity.into()), diff --git a/polars/polars-arrow/src/kernels/rolling/nulls/quantile.rs b/polars/polars-arrow/src/kernels/rolling/nulls/quantile.rs index ed521805d236..c4978e79b7fa 100644 --- a/polars/polars-arrow/src/kernels/rolling/nulls/quantile.rs +++ b/polars/polars-arrow/src/kernels/rolling/nulls/quantile.rs @@ -29,7 +29,7 @@ where { if values.is_empty() { let out: Vec = vec![]; - return Arc::new(PrimitiveArray::from_data( + return Box::new(PrimitiveArray::from_data( T::PRIMITIVE.into(), out.into(), None, @@ -70,7 +70,7 @@ where }) .collect_trusted::>(); - Arc::new(PrimitiveArray::from_data( + Box::new(PrimitiveArray::from_data( T::PRIMITIVE.into(), out.into(), Some(validity.into()), @@ -134,7 +134,7 @@ where }) .collect_trusted::>(); - Arc::new(PrimitiveArray::from_data( + Box::new(PrimitiveArray::from_data( T::PRIMITIVE.into(), out.into(), Some(validity.into()), diff --git a/polars/polars-arrow/src/kernels/string.rs b/polars/polars-arrow/src/kernels/string.rs index 34395d27f349..ec5b38045850 100644 --- a/polars/polars-arrow/src/kernels/string.rs +++ b/polars/polars-arrow/src/kernels/string.rs @@ -1,8 +1,8 @@ +use crate::prelude::*; use crate::trusted_len::PushUnchecked; -use arrow::array::{ArrayRef, UInt32Array, Utf8Array}; +use arrow::array::{UInt32Array, Utf8Array}; use arrow::buffer::Buffer; use arrow::datatypes::DataType; -use std::sync::Arc; pub fn string_lengths(array: &Utf8Array) -> ArrayRef { let values = array.offsets().windows(2).map(|x| (x[1] - x[0]) as u32); @@ -10,5 +10,5 @@ pub fn string_lengths(array: &Utf8Array) -> ArrayRef { let values: Buffer<_> = Vec::from_trusted_len_iter(values).into(); let array = UInt32Array::from_data(DataType::UInt32, values, array.validity().cloned()); - Arc::new(array) + Box::new(array) } diff --git a/polars/polars-arrow/src/kernels/take.rs b/polars/polars-arrow/src/kernels/take.rs index ece851dc1def..59c90148fa09 100644 --- a/polars/polars-arrow/src/kernels/take.rs +++ b/polars/polars-arrow/src/kernels/take.rs @@ -6,7 +6,6 @@ use arrow::bitmap::MutableBitmap; use arrow::buffer::Buffer; use arrow::datatypes::{DataType, PhysicalType}; use arrow::types::NativeType; -use std::sync::Arc; /// # Safety /// Does not do bounds checks @@ -29,7 +28,7 @@ pub unsafe fn take_unchecked(arr: &dyn Array, idx: &IdxArr) -> ArrayRef { #[cfg(feature = "compute")] _ => { use arrow::compute::take::take; - Arc::from(take(arr, idx).unwrap()) + take(arr, idx).unwrap() } #[cfg(not(feature = "compute"))] _ => { @@ -44,7 +43,7 @@ pub unsafe fn take_unchecked(arr: &dyn Array, idx: &IdxArr) -> ArrayRef { pub unsafe fn take_primitive_unchecked( arr: &PrimitiveArray, indices: &IdxArr, -) -> Arc> { +) -> Box> { let array_values = arr.values().as_slice(); let index_values = indices.values().as_slice(); let validity_values = arr.validity().expect("should have nulls"); @@ -85,7 +84,7 @@ pub unsafe fn take_primitive_unchecked( }; let arr = PrimitiveArray::from_data(T::PRIMITIVE.into(), values.into(), Some(validity.into())); - Arc::new(arr) + Box::new(arr) } /// Take kernel for single chunk without nulls and arrow array as index. @@ -94,7 +93,7 @@ pub unsafe fn take_primitive_unchecked( pub unsafe fn take_no_null_primitive( arr: &PrimitiveArray, indices: &IdxArr, -) -> Arc> { +) -> Box> { debug_assert!(!arr.has_validity()); let array_values = arr.values().as_slice(); let index_values = indices.values().as_slice(); @@ -106,7 +105,7 @@ pub unsafe fn take_no_null_primitive( let values: Buffer<_> = Vec::from_trusted_len_iter(iter).into(); let validity = indices.validity().cloned(); - Arc::new(PrimitiveArray::from_data( + Box::new(PrimitiveArray::from_data( T::PRIMITIVE.into(), values, validity, @@ -122,7 +121,7 @@ pub unsafe fn take_no_null_primitive( pub unsafe fn take_no_null_primitive_iter_unchecked>( arr: &PrimitiveArray, indices: I, -) -> Arc> { +) -> Box> { debug_assert!(!arr.has_validity()); let array_values = arr.values().as_slice(); @@ -132,7 +131,7 @@ pub unsafe fn take_no_null_primitive_iter_unchecked = Vec::from_trusted_len_iter(iter).into(); - Arc::new(PrimitiveArray::from_data(T::PRIMITIVE.into(), values, None)) + Box::new(PrimitiveArray::from_data(T::PRIMITIVE.into(), values, None)) } /// Take kernel for a single chunk with null values and an iterator as index. @@ -144,7 +143,7 @@ pub unsafe fn take_no_null_primitive_iter_unchecked>( arr: &PrimitiveArray, indices: I, -) -> Arc> { +) -> Box> { let array_values = arr.values().as_slice(); let validity = arr.validity().expect("should have nulls"); @@ -157,7 +156,7 @@ pub unsafe fn take_primitive_iter_unchecked( arr: &PrimitiveArray, indices: I, -) -> Arc> { +) -> Box> { let array_values = arr.values().as_slice(); let iter = indices.into_iter().map(|opt_idx| { @@ -184,7 +183,7 @@ pub unsafe fn take_no_null_primitive_opt_iter_unchecked< }); let arr = PrimitiveArray::from_trusted_len_iter_unchecked(iter).to(T::PRIMITIVE.into()); - Arc::new(arr) + Box::new(arr) } /// Take kernel for a single chunk and an iterator that can produce None values. @@ -200,7 +199,7 @@ pub unsafe fn take_primitive_opt_iter_unchecked< >( arr: &PrimitiveArray, indices: I, -) -> Arc> { +) -> Box> { let array_values = arr.values().as_slice(); let validity = arr.validity().expect("should have nulls"); @@ -216,7 +215,7 @@ pub unsafe fn take_primitive_opt_iter_unchecked< }); let arr = PrimitiveArray::from_trusted_len_iter_unchecked(iter).to(T::PRIMITIVE.into()); - Arc::new(arr) + Box::new(arr) } /// Take kernel for single chunk without nulls and an iterator as index. @@ -228,7 +227,7 @@ pub unsafe fn take_primitive_opt_iter_unchecked< pub unsafe fn take_no_null_bool_iter_unchecked>( arr: &BooleanArray, indices: I, -) -> Arc { +) -> Box { debug_assert!(!arr.has_validity()); let values = arr.values(); @@ -237,7 +236,7 @@ pub unsafe fn take_no_null_bool_iter_unchecked>( values.get_bit_unchecked(idx) }); let mutable = MutableBitmap::from_trusted_len_iter_unchecked(iter); - Arc::new(BooleanArray::from_data( + Box::new(BooleanArray::from_data( DataType::Boolean, mutable.into(), None, @@ -252,7 +251,7 @@ pub unsafe fn take_no_null_bool_iter_unchecked>( pub unsafe fn take_bool_iter_unchecked>( arr: &BooleanArray, indices: I, -) -> Arc { +) -> Box { let validity = arr.validity().expect("should have nulls"); let iter = indices.into_iter().map(|idx| { @@ -263,7 +262,7 @@ pub unsafe fn take_bool_iter_unchecked>( } }); - Arc::new(BooleanArray::from_trusted_len_iter_unchecked(iter)) + Box::new(BooleanArray::from_trusted_len_iter_unchecked(iter)) } /// Take kernel for single chunk and an iterator as index. @@ -274,7 +273,7 @@ pub unsafe fn take_bool_iter_unchecked>( pub unsafe fn take_bool_opt_iter_unchecked>>( arr: &BooleanArray, indices: I, -) -> Arc { +) -> Box { let validity = arr.validity().expect("should have nulls"); let iter = indices.into_iter().map(|opt_idx| { opt_idx.and_then(|idx| { @@ -286,7 +285,7 @@ pub unsafe fn take_bool_opt_iter_unchecked> }) }); - Arc::new(BooleanArray::from_trusted_len_iter_unchecked(iter)) + Box::new(BooleanArray::from_trusted_len_iter_unchecked(iter)) } /// Take kernel for single chunk without null values and an iterator as index that may produce None values. @@ -297,12 +296,12 @@ pub unsafe fn take_bool_opt_iter_unchecked> pub unsafe fn take_no_null_bool_opt_iter_unchecked>>( arr: &BooleanArray, indices: I, -) -> Arc { +) -> Box { let iter = indices .into_iter() .map(|opt_idx| opt_idx.map(|idx| arr.value_unchecked(idx))); - Arc::new(BooleanArray::from_trusted_len_iter_unchecked(iter)) + Box::new(BooleanArray::from_trusted_len_iter_unchecked(iter)) } /// # Safety @@ -312,12 +311,12 @@ pub unsafe fn take_no_null_bool_opt_iter_unchecked>( arr: &LargeStringArray, indices: I, -) -> Arc { +) -> Box { let iter = indices.into_iter().map(|idx| { debug_assert!(idx < arr.len()); arr.value_unchecked(idx) }); - Arc::new(MutableUtf8Array::::from_trusted_len_values_iter_unchecked(iter).into()) + Box::new(MutableUtf8Array::::from_trusted_len_values_iter_unchecked(iter).into()) } /// # Safety @@ -327,7 +326,7 @@ pub unsafe fn take_no_null_utf8_iter_unchecked>( pub unsafe fn take_utf8_iter_unchecked>( arr: &LargeStringArray, indices: I, -) -> Arc { +) -> Box { let validity = arr.validity().expect("should have nulls"); let iter = indices.into_iter().map(|idx| { debug_assert!(idx < arr.len()); @@ -338,7 +337,7 @@ pub unsafe fn take_utf8_iter_unchecked>( } }); - Arc::new(LargeStringArray::from_trusted_len_iter_unchecked(iter)) + Box::new(LargeStringArray::from_trusted_len_iter_unchecked(iter)) } /// # Safety @@ -348,12 +347,12 @@ pub unsafe fn take_utf8_iter_unchecked>( pub unsafe fn take_no_null_utf8_opt_iter_unchecked>>( arr: &LargeStringArray, indices: I, -) -> Arc { +) -> Box { let iter = indices .into_iter() .map(|opt_idx| opt_idx.map(|idx| arr.value_unchecked(idx))); - Arc::new(LargeStringArray::from_trusted_len_iter_unchecked(iter)) + Box::new(LargeStringArray::from_trusted_len_iter_unchecked(iter)) } /// # Safety @@ -363,7 +362,7 @@ pub unsafe fn take_no_null_utf8_opt_iter_unchecked>>( arr: &LargeStringArray, indices: I, -) -> Arc { +) -> Box { let validity = arr.validity().expect("should have nulls"); let iter = indices.into_iter().map(|opt_idx| { opt_idx.and_then(|idx| { @@ -374,7 +373,7 @@ pub unsafe fn take_utf8_opt_iter_unchecked> } }) }); - Arc::new(LargeStringArray::from_trusted_len_iter_unchecked(iter)) + Box::new(LargeStringArray::from_trusted_len_iter_unchecked(iter)) } /// # Safety @@ -382,7 +381,7 @@ pub unsafe fn take_utf8_opt_iter_unchecked> pub unsafe fn take_utf8_unchecked( arr: &LargeStringArray, indices: &IdxArr, -) -> Arc { +) -> Box { let data_len = indices.len(); let mut offset_buf = vec![0; data_len + 1]; @@ -479,11 +478,11 @@ pub unsafe fn take_utf8_unchecked( } let array: Utf8Array = builder.into(); - return Arc::new(array); + return Box::new(array); } // Safety: all "values" are &str, and thus valid utf8 - Arc::new(Utf8Array::::from_data_unchecked_default( + Box::new(Utf8Array::::from_data_unchecked_default( offset_buf.into(), values_buf.into(), validity, diff --git a/polars/polars-arrow/src/prelude.rs b/polars/polars-arrow/src/prelude.rs index 66f2d1ae4cc2..da204a426c8f 100644 --- a/polars/polars-arrow/src/prelude.rs +++ b/polars/polars-arrow/src/prelude.rs @@ -1,7 +1,7 @@ pub use crate::array::default_arrays::*; pub use crate::bitmap::mutable::MutableBitmapExtension; pub use crate::kernels::rolling::no_nulls::QuantileInterpolOptions; -pub use crate::{array::*, index::*}; +pub use crate::{array::*, data_types::*, index::*}; use arrow::array::{ListArray, Utf8Array}; pub type LargeStringArray = Utf8Array; diff --git a/polars/polars-arrow/src/utils.rs b/polars/polars-arrow/src/utils.rs index dcbf9943520a..469ee7850ad4 100644 --- a/polars/polars-arrow/src/utils.rs +++ b/polars/polars-arrow/src/utils.rs @@ -182,6 +182,7 @@ macro_rules! with_match_primitive_type {( UInt64 => __with_ty__! { u64 }, Float32 => __with_ty__! { f32 }, Float64 => __with_ty__! { f64 }, + Float16 => unimplemented!(), } })} pub(crate) use with_match_primitive_type; diff --git a/polars/polars-core/Cargo.toml b/polars/polars-core/Cargo.toml index c171dfadf82c..513a4fba4728 100644 --- a/polars/polars-core/Cargo.toml +++ b/polars/polars-core/Cargo.toml @@ -173,9 +173,9 @@ thiserror = "^1.0" package = "arrow2" git = "https://github.com/jorgecarleitao/arrow2" # git = "https://github.com/ritchie46/arrow2" -rev = "6919cc3281c3ee7dd3e973a7b303c67277458f20" +rev = "39db6fb7514364bfea08d594793b23e1ed5a7def" # path = "../../../arrow2" -# branch = "improve_mutable" +# branch = "count_shared" # version = "0.12" default-features = false features = [ diff --git a/polars/polars-core/src/chunked_array/arithmetic.rs b/polars/polars-core/src/chunked_array/arithmetic.rs index 030f5dcd3ba4..1f847202116e 100644 --- a/polars/polars-core/src/chunked_array/arithmetic.rs +++ b/polars/polars-core/src/chunked_array/arithmetic.rs @@ -2,11 +2,10 @@ use crate::prelude::*; use crate::utils::align_chunks_binary; use arrow::array::PrimitiveArray; -use arrow::{array::ArrayRef, compute, compute::arithmetics::basic}; +use arrow::{compute, compute::arithmetics::basic}; use num::{Num, NumCast, ToPrimitive}; use std::borrow::Cow; use std::ops::{Add, Div, Mul, Rem, Sub}; -use std::sync::Arc; macro_rules! apply_operand_on_chunkedarray_by_iter { @@ -69,7 +68,7 @@ where let chunks = lhs .downcast_iter() .zip(rhs.downcast_iter()) - .map(|(lhs, rhs)| Arc::new(kernel(lhs, rhs)) as ArrayRef) + .map(|(lhs, rhs)| Box::new(kernel(lhs, rhs)) as ArrayRef) .collect(); lhs.copy_with_chunks(chunks) } @@ -243,7 +242,7 @@ where fn div(self, rhs: N) -> Self::Output { let rhs: T::Native = NumCast::from(rhs).expect("could not cast"); - self.apply_kernel(&|arr| Arc::new(basic::div_scalar(arr, &rhs))) + self.apply_kernel(&|arr| Box::new(basic::div_scalar(arr, &rhs))) } } @@ -269,7 +268,7 @@ where fn rem(self, rhs: N) -> Self::Output { let rhs: T::Native = NumCast::from(rhs).expect("could not cast"); - self.apply_kernel(&|arr| Arc::new(basic::rem_scalar(arr, &rhs))) + self.apply_kernel(&|arr| Box::new(basic::rem_scalar(arr, &rhs))) } } @@ -420,7 +419,7 @@ where fn pow_f32(&self, exp: f32) -> Float32Chunked { let s = self.cast(&DataType::Float32).unwrap(); s.f32().unwrap().apply_kernel(&|arr| { - Arc::new(compute::arity::unary( + Box::new(compute::arity::unary( arr, |x| x.powf(exp), DataType::Float32.to_arrow(), @@ -431,7 +430,7 @@ where fn pow_f64(&self, exp: f64) -> Float64Chunked { let s = self.cast(&DataType::Float64).unwrap(); s.f64().unwrap().apply_kernel(&|arr| { - Arc::new(compute::arity::unary( + Box::new(compute::arity::unary( arr, |x| x.powf(exp), DataType::Float64.to_arrow(), diff --git a/polars/polars-core/src/chunked_array/bitwise.rs b/polars/polars-core/src/chunked_array/bitwise.rs index f5e6f6f5b242..372d5d35f8c2 100644 --- a/polars/polars-core/src/chunked_array/bitwise.rs +++ b/polars/polars-core/src/chunked_array/bitwise.rs @@ -27,7 +27,7 @@ where .collect_trusted::>(); let arr = PrimitiveArray::from_data(T::get_dtype().to_arrow(), av.into(), validity); - Arc::new(arr) as ArrayRef + Box::new(arr) as ArrayRef }) .collect::>(); @@ -59,7 +59,7 @@ where .collect_trusted::>(); let arr = PrimitiveArray::from_data(T::get_dtype().to_arrow(), av.into(), validity); - Arc::new(arr) as ArrayRef + Box::new(arr) as ArrayRef }) .collect::>(); @@ -91,7 +91,7 @@ where .collect_trusted::>(); let arr = PrimitiveArray::from_data(T::get_dtype().to_arrow(), av.into(), validity); - Arc::new(arr) as ArrayRef + Box::new(arr) as ArrayRef }) .collect::>(); @@ -133,7 +133,7 @@ impl BitOr for &BooleanChunked { .downcast_iter() .zip(rhs.downcast_iter()) .map(|(lhs, rhs)| { - Arc::new(compute::boolean_kleene::or(lhs, rhs).expect("should be same size")) + Box::new(compute::boolean_kleene::or(lhs, rhs).expect("should be same size")) as ArrayRef }) .collect(); @@ -191,7 +191,7 @@ impl BitXor for &BooleanChunked { let values = l_arr.values() ^ r_arr.values(); let arr = BooleanArray::from_data_default(values, validity); - Arc::new(arr) as ArrayRef + Box::new(arr) as ArrayRef }) .collect::>(); @@ -237,7 +237,7 @@ impl BitAnd for &BooleanChunked { .downcast_iter() .zip(rhs.downcast_iter()) .map(|(lhs, rhs)| { - Arc::new(compute::boolean_kleene::and(lhs, rhs).expect("should be same size")) + Box::new(compute::boolean_kleene::and(lhs, rhs).expect("should be same size")) as ArrayRef }) .collect(); diff --git a/polars/polars-core/src/chunked_array/builder/boolean.rs b/polars/polars-core/src/chunked_array/builder/boolean.rs index a010425a0e33..897b7f9a9a92 100644 --- a/polars/polars-core/src/chunked_array/builder/boolean.rs +++ b/polars/polars-core/src/chunked_array/builder/boolean.rs @@ -20,7 +20,7 @@ impl ChunkedBuilder for BooleanChunkedBuilder { fn finish(self) -> BooleanChunked { let arr: BooleanArray = self.array_builder.into(); - let arr = Arc::new(arr) as ArrayRef; + let arr = Box::new(arr) as ArrayRef; ChunkedArray { field: Arc::new(self.field), diff --git a/polars/polars-core/src/chunked_array/builder/from.rs b/polars/polars-core/src/chunked_array/builder/from.rs index 4b83927efc56..f53212a84d3f 100644 --- a/polars/polars-core/src/chunked_array/builder/from.rs +++ b/polars/polars-core/src/chunked_array/builder/from.rs @@ -1,13 +1,12 @@ use crate::prelude::*; use arrow::array::{BooleanArray, PrimitiveArray, Utf8Array}; -use std::sync::Arc; impl From<(&str, PrimitiveArray)> for ChunkedArray { fn from(tpl: (&str, PrimitiveArray)) -> Self { let name = tpl.0; let arr = tpl.1; - ChunkedArray::from_chunks(name, vec![Arc::new(arr)]) + ChunkedArray::from_chunks(name, vec![Box::new(arr)]) } } @@ -22,13 +21,13 @@ impl From<(&str, BooleanArray)> for BooleanChunked { let name = tpl.0; let arr = tpl.1; - ChunkedArray::from_chunks(name, vec![Arc::new(arr)]) + ChunkedArray::from_chunks(name, vec![Box::new(arr)]) } } impl From for BooleanChunked { fn from(arr: BooleanArray) -> Self { - ChunkedArray::from_chunks("", vec![Arc::new(arr)]) + ChunkedArray::from_chunks("", vec![Box::new(arr)]) } } @@ -37,6 +36,6 @@ impl From<(&str, Utf8Array)> for Utf8Chunked { let name = tpl.0; let arr = tpl.1; - ChunkedArray::from_chunks(name, vec![Arc::new(arr)]) + ChunkedArray::from_chunks(name, vec![Box::new(arr)]) } } diff --git a/polars/polars-core/src/chunked_array/builder/list.rs b/polars/polars-core/src/chunked_array/builder/list.rs index b480caa457db..ba2ebeee51e9 100644 --- a/polars/polars-core/src/chunked_array/builder/list.rs +++ b/polars/polars-core/src/chunked_array/builder/list.rs @@ -45,7 +45,7 @@ where macro_rules! finish_list_builder { ($self:ident) => {{ - let arr = $self.builder.as_arc(); + let arr = $self.builder.as_box(); let mut ca = ListChunked { field: Arc::new($self.field.clone()), chunks: vec![arr], @@ -477,7 +477,7 @@ impl<'a> AnonymousListBuilder<'a> { let dtype = slf.dtype.map(|dt| dt.to_physical().to_arrow()); let arr = slf.builder.finish(dtype.as_ref()).unwrap(); let dtype = DataType::from(arr.data_type()); - let mut ca = ListChunked::from_chunks("", vec![Arc::new(arr)]); + let mut ca = ListChunked::from_chunks("", vec![Box::new(arr)]); ca.field = Arc::new(Field::new(&slf.name, dtype)); ca @@ -539,14 +539,9 @@ impl ListBuilderTrait for AnonymousOwnedListBuilder { let array = new_null_array(dtype.clone(), real_length); let dtype = ListArray::::default_datatype(dtype); let array = unsafe { - ListArray::new_unchecked( - dtype, - slf.builder.take_offsets().into(), - Arc::from(array), - None, - ) + ListArray::new_unchecked(dtype, slf.builder.take_offsets().into(), array, None) }; - ListChunked::from_chunks(&slf.name, vec![Arc::new(array)]) + ListChunked::from_chunks(&slf.name, vec![Box::new(array)]) } else { ListChunked::full_null_with_dtype( &slf.name, @@ -558,7 +553,7 @@ impl ListBuilderTrait for AnonymousOwnedListBuilder { let inner_dtype = slf.inner_dtype.map(|dt| dt.to_physical().to_arrow()); let arr = slf.builder.finish(inner_dtype.as_ref()).unwrap(); let dtype = DataType::from(arr.data_type()); - let mut ca = ListChunked::from_chunks("", vec![Arc::new(arr)]); + let mut ca = ListChunked::from_chunks("", vec![Box::new(arr)]); ca.field = Arc::new(Field::new(&slf.name, dtype)); ca diff --git a/polars/polars-core/src/chunked_array/builder/mod.rs b/polars/polars-core/src/chunked_array/builder/mod.rs index 841df7144e37..7e784625320a 100644 --- a/polars/polars-core/src/chunked_array/builder/mod.rs +++ b/polars/polars-core/src/chunked_array/builder/mod.rs @@ -62,7 +62,7 @@ where { fn from_slice(name: &str, v: &[T::Native]) -> Self { let arr = PrimitiveArray::::from_slice(v).to(T::get_dtype().to_arrow()); - ChunkedArray::from_chunks(name, vec![Arc::new(arr)]) + ChunkedArray::from_chunks(name, vec![Box::new(arr)]) } fn from_slice_options(name: &str, opt_v: &[Option]) -> Self { @@ -127,7 +127,7 @@ where ChunkedArray { field, - chunks: vec![builder.into_arc()], + chunks: vec![builder.as_box()], phantom: PhantomData, categorical_map: None, ..Default::default() @@ -146,7 +146,7 @@ where ChunkedArray { field, - chunks: vec![builder.into_arc()], + chunks: vec![builder.as_box()], phantom: PhantomData, categorical_map: None, ..Default::default() diff --git a/polars/polars-core/src/chunked_array/builder/primitive.rs b/polars/polars-core/src/chunked_array/builder/primitive.rs index 5b48e95d1069..b933ec5f62a9 100644 --- a/polars/polars-core/src/chunked_array/builder/primitive.rs +++ b/polars/polars-core/src/chunked_array/builder/primitive.rs @@ -24,10 +24,10 @@ where self.array_builder.push(None) } - fn finish(self) -> ChunkedArray { + fn finish(mut self) -> ChunkedArray { ChunkedArray { field: Arc::new(self.field), - chunks: vec![self.array_builder.into_arc()], + chunks: vec![self.array_builder.as_box()], phantom: PhantomData, categorical_map: None, ..Default::default() diff --git a/polars/polars-core/src/chunked_array/builder/utf8.rs b/polars/polars-core/src/chunked_array/builder/utf8.rs index 7953c9ae7e3b..08282570aa77 100644 --- a/polars/polars-core/src/chunked_array/builder/utf8.rs +++ b/polars/polars-core/src/chunked_array/builder/utf8.rs @@ -38,8 +38,8 @@ impl Utf8ChunkedBuilder { self.builder.push(opt); } - pub fn finish(self) -> Utf8Chunked { - let arr = self.builder.into_arc(); + pub fn finish(mut self) -> Utf8Chunked { + let arr = self.builder.as_box(); ChunkedArray { field: Arc::new(self.field), chunks: vec![arr], diff --git a/polars/polars-core/src/chunked_array/cast.rs b/polars/polars-core/src/chunked_array/cast.rs index 23053e7b7b8d..a260733ef24e 100644 --- a/polars/polars-core/src/chunked_array/cast.rs +++ b/polars/polars-core/src/chunked_array/cast.rs @@ -9,7 +9,6 @@ pub(crate) fn cast_chunks(chunks: &[ArrayRef], dtype: &DataType) -> Result>>()?; Ok(chunks) } @@ -102,11 +101,15 @@ impl ChunkCast for BooleanChunked { fn cast_inner_list_type(list: &ListArray, child_type: &DataType) -> Result { let child = list.values(); let offsets = list.offsets(); - let child = cast::cast(child.as_ref(), &child_type.to_arrow())?.into(); + let child = cast::cast(child.as_ref(), &child_type.to_arrow())?; let data_type = ListArray::::default_datatype(child_type.to_arrow()); - let list = ListArray::from_data(data_type, offsets.clone(), child, list.validity().cloned()); - Ok(Arc::new(list) as ArrayRef) + // Safety: + // offsets are correct as they have not changed + let list = unsafe { + ListArray::new_unchecked(data_type, offsets.clone(), child, list.validity().cloned()) + }; + Ok(Box::new(list) as ArrayRef) } /// We cannot cast anything to or from List/LargeList diff --git a/polars/polars-core/src/chunked_array/comparison.rs b/polars/polars-core/src/chunked_array/comparison.rs index b98c12b66f65..60fd157e32a5 100644 --- a/polars/polars-core/src/chunked_array/comparison.rs +++ b/polars/polars-core/src/chunked_array/comparison.rs @@ -2,14 +2,13 @@ use crate::utils::align_chunks_binary; use crate::{prelude::*, utils::NoNull}; use arrow::scalar::{PrimitiveScalar, Scalar, Utf8Scalar}; use arrow::{ - array::{ArrayRef, BooleanArray, PrimitiveArray, Utf8Array}, + array::{BooleanArray, PrimitiveArray, Utf8Array}, compute, compute::comparison, }; use num::{NumCast, ToPrimitive}; use polars_arrow::prelude::FromData; use std::ops::Not; -use std::sync::Arc; impl ChunkedArray where @@ -27,7 +26,7 @@ where .zip(rhs.downcast_iter()) .map(|(left, right)| { let arr = f(left, right); - Arc::new(arr) as ArrayRef + Box::new(arr) as ArrayRef }) .collect::>(); @@ -221,7 +220,7 @@ fn compare_bools( let chunks = lhs .downcast_iter() .zip(rhs.downcast_iter()) - .map(|(l, r)| Arc::new(f(l, r)) as ArrayRef) + .map(|(l, r)| Box::new(f(l, r)) as ArrayRef) .collect(); BooleanChunked::from_chunks(lhs.name(), chunks) @@ -248,13 +247,13 @@ impl ChunkCompare<&BooleanChunked> for BooleanChunked { .downcast_iter() .map(|arr| { if let Some(validity) = arr.validity() { - Arc::new(BooleanArray::from_data_default( + Box::new(BooleanArray::from_data_default( arr.values() & validity, None, )) as ArrayRef } else { - Arc::new(arr.clone()) + Box::new(arr.clone()) } }) .collect(); @@ -273,7 +272,7 @@ impl ChunkCompare<&BooleanChunked> for BooleanChunked { } else { arr.values().not() }; - Arc::new(BooleanArray::from_data_default(bitmap, None)) + Box::new(BooleanArray::from_data_default(bitmap, None)) as ArrayRef }) .collect(); @@ -312,7 +311,7 @@ impl ChunkCompare<&BooleanChunked> for BooleanChunked { } else { arr.values().not() }; - Arc::new(BooleanArray::from_data_default(bitmap, None)) + Box::new(BooleanArray::from_data_default(bitmap, None)) as ArrayRef }) .collect(); @@ -331,7 +330,7 @@ impl ChunkCompare<&BooleanChunked> for BooleanChunked { } else { arr.values().clone() }; - Arc::new(BooleanArray::from_data_default(bitmap, None)) + Box::new(BooleanArray::from_data_default(bitmap, None)) as ArrayRef }) .collect(); @@ -490,7 +489,7 @@ impl Utf8Chunked { .zip(rhs.downcast_iter()) .map(|(left, right)| { let arr = f(left, right); - Arc::new(arr) as ArrayRef + Box::new(arr) as ArrayRef }) .collect(); BooleanChunked::from_chunks("", chunks) @@ -649,7 +648,7 @@ where let rhs: T::Native = NumCast::from(rhs).expect("could not cast to underlying chunkedarray type"); let scalar = PrimitiveScalar::new(T::get_dtype().to_arrow(), Some(rhs)); - self.apply_kernel_cast(&|arr| Arc::new(f(arr, &scalar))) + self.apply_kernel_cast(&|arr| Box::new(f(arr, &scalar))) } } @@ -695,7 +694,7 @@ impl Utf8Chunked { f: impl Fn(&Utf8Array, &dyn Scalar) -> BooleanArray, ) -> BooleanChunked { let scalar = Utf8Scalar::::new(Some(rhs)); - self.apply_kernel_cast(&|arr| Arc::new(f(arr, &scalar))) + self.apply_kernel_cast(&|arr| Box::new(f(arr, &scalar))) } } @@ -827,7 +826,7 @@ impl Not for &BooleanChunked { .downcast_iter() .map(|a| { let arr = compute::boolean::not(a); - Arc::new(arr) as ArrayRef + Box::new(arr) as ArrayRef }) .collect::>(); ChunkedArray::from_chunks(self.name(), chunks) diff --git a/polars/polars-core/src/chunked_array/float.rs b/polars/polars-core/src/chunked_array/float.rs index 557b8167be9f..456bc1c1547e 100644 --- a/polars/polars-core/src/chunked_array/float.rs +++ b/polars/polars-core/src/chunked_array/float.rs @@ -26,7 +26,7 @@ where pub fn none_to_nan(&self) -> Self { let chunks = self .downcast_iter() - .map(|arr| Arc::new(set_at_nulls(arr, T::Native::nan())) as ArrayRef) + .map(|arr| Box::new(set_at_nulls(arr, T::Native::nan())) as ArrayRef) .collect(); ChunkedArray::from_chunks(self.name(), chunks) } diff --git a/polars/polars-core/src/chunked_array/iterator/mod.rs b/polars/polars-core/src/chunked_array/iterator/mod.rs index 29686364e47a..68149855ae45 100644 --- a/polars/polars-core/src/chunked_array/iterator/mod.rs +++ b/polars/polars-core/src/chunked_array/iterator/mod.rs @@ -221,11 +221,7 @@ impl<'a> IntoIterator for &'a ListChunked { .trust_my_length(self.len()) .map(move |arr| { arr.map(|arr| { - Series::from_chunks_and_dtype_unchecked( - "", - vec![Arc::from(arr)], - &dtype, - ) + Series::from_chunks_and_dtype_unchecked("", vec![arr], &dtype) }) }), ) @@ -264,7 +260,7 @@ impl<'a> Iterator for ListIterNoNull<'a> { unsafe { Some(Series::from_chunks_and_dtype_unchecked( "", - vec![Arc::from(self.array.value_unchecked(old))], + vec![self.array.value_unchecked(old)], &self.inner_type, )) } diff --git a/polars/polars-core/src/chunked_array/iterator/par/list.rs b/polars/polars-core/src/chunked_array/iterator/par/list.rs index 0daa594fa083..04e9c3738349 100644 --- a/polars/polars-core/src/chunked_array/iterator/par/list.rs +++ b/polars/polars-core/src/chunked_array/iterator/par/list.rs @@ -3,7 +3,7 @@ use rayon::prelude::*; unsafe fn idx_to_array(idx: usize, arr: &ListArray, dtype: &DataType) -> Option { if arr.is_valid(idx) { - Some(Arc::from(arr.value_unchecked(idx))) + Some(arr.value_unchecked(idx)) .map(|arr: ArrayRef| Series::from_chunks_and_dtype_unchecked("", vec![arr], dtype)) } else { None diff --git a/polars/polars-core/src/chunked_array/kernels/take.rs b/polars/polars-core/src/chunked_array/kernels/take.rs index a0af00d3bf38..916b249dab88 100644 --- a/polars/polars-core/src/chunked_array/kernels/take.rs +++ b/polars/polars-core/src/chunked_array/kernels/take.rs @@ -47,7 +47,7 @@ pub(crate) unsafe fn take_list_unchecked( let taken = s .take_unchecked(&IdxCa::from_chunks( "", - vec![Arc::new(list_indices) as ArrayRef], + vec![Box::new(list_indices) as ArrayRef], )) .unwrap(); @@ -75,5 +75,7 @@ pub(crate) unsafe fn take_list_unchecked( } else { None }; - ListArray::from_data(values.data_type().clone(), offsets.into(), taken, validity) + // Safety: + // offsets are monotonically increasing + ListArray::new_unchecked(values.data_type().clone(), offsets.into(), taken, validity) } diff --git a/polars/polars-core/src/chunked_array/list/iterator.rs b/polars/polars-core/src/chunked_array/list/iterator.rs index 13363d6207a7..1c4149151bdd 100644 --- a/polars/polars-core/src/chunked_array/list/iterator.rs +++ b/polars/polars-core/src/chunked_array/list/iterator.rs @@ -1,7 +1,6 @@ use crate::prelude::*; use crate::series::unstable::{ArrayBox, UnstableSeries}; use crate::utils::CustomIterTools; -use arrow::array::ArrayRef; use std::marker::PhantomData; use std::ptr::NonNull; @@ -30,7 +29,6 @@ impl<'a, I: Iterator>> Iterator for AmortizedListIter<'a // Safety // dtype is known unsafe { - let array_ref = Arc::from(array_ref); let mut s = Series::from_chunks_and_dtype_unchecked( "", vec![array_ref], @@ -45,7 +43,7 @@ impl<'a, I: Iterator>> Iterator for AmortizedListIter<'a } // update the inner state - unsafe { *self.inner.as_mut() = array_ref.into() }; + unsafe { *self.inner.as_mut() = array_ref }; // Safety // we cannot control the lifetime of an iterators `next` method. diff --git a/polars/polars-core/src/chunked_array/logical/categorical/builder.rs b/polars/polars-core/src/chunked_array/logical/categorical/builder.rs index ae2d217b13ef..afb6d93235f4 100644 --- a/polars/polars-core/src/chunked_array/logical/categorical/builder.rs +++ b/polars/polars-core/src/chunked_array/logical/categorical/builder.rs @@ -201,10 +201,10 @@ impl CategoricalChunkedBuilder { } } - pub fn finish(self) -> CategoricalChunked { + pub fn finish(mut self) -> CategoricalChunked { CategoricalChunked::from_chunks_original( &self.name, - vec![self.array_builder.into_arc()], + vec![self.array_builder.as_box()], self.reverse_mapping.finish(), ) } diff --git a/polars/polars-core/src/chunked_array/logical/categorical/from.rs b/polars/polars-core/src/chunked_array/logical/categorical/from.rs index 515bb94c6543..f4a059a3b05f 100644 --- a/polars/polars-core/src/chunked_array/logical/categorical/from.rs +++ b/polars/polars-core/src/chunked_array/logical/categorical/from.rs @@ -9,7 +9,7 @@ impl From<&CategoricalChunked> for DictionaryArray { let map = &**ca.get_rev_map(); match map { RevMapping::Local(arr) => { - DictionaryArray::from_data(keys.clone(), Arc::new(arr.clone())) + DictionaryArray::from_data(keys.clone(), Box::new(arr.clone())) } RevMapping::Global(reverse_map, values, _uuid) => { let iter = keys @@ -17,7 +17,7 @@ impl From<&CategoricalChunked> for DictionaryArray { .map(|opt_k| opt_k.map(|k| *reverse_map.get(k).unwrap())); let keys = PrimitiveArray::from_trusted_len_iter(iter); - DictionaryArray::from_data(keys, Arc::new(values.clone())) + DictionaryArray::from_data(keys, Box::new(values.clone())) } } } @@ -35,7 +35,7 @@ impl From<&CategoricalChunked> for DictionaryArray { .downcast_ref::>() .unwrap() .clone(), - Arc::new(arr.clone()), + Box::new(arr.clone()), ), RevMapping::Global(reverse_map, values, _uuid) => { let iter = keys @@ -43,7 +43,7 @@ impl From<&CategoricalChunked> for DictionaryArray { .map(|opt_k| opt_k.map(|k| *reverse_map.get(k).unwrap() as i64)); let keys = PrimitiveArray::from_trusted_len_iter(iter); - DictionaryArray::from_data(keys, Arc::new(values.clone())) + DictionaryArray::from_data(keys, Box::new(values.clone())) } } } diff --git a/polars/polars-core/src/chunked_array/logical/categorical/mod.rs b/polars/polars-core/src/chunked_array/logical/categorical/mod.rs index c722398535f7..6bc28a31d8e0 100644 --- a/polars/polars-core/src/chunked_array/logical/categorical/mod.rs +++ b/polars/polars-core/src/chunked_array/logical/categorical/mod.rs @@ -217,7 +217,7 @@ mod test { let ca = ca.categorical().unwrap(); let arr: DictionaryArray = (ca).into(); - let s = Series::try_from(("foo", Arc::new(arr) as ArrayRef))?; + let s = Series::try_from(("foo", Box::new(arr) as ArrayRef))?; assert!(matches!(s.dtype(), &DataType::Categorical(_))); assert_eq!(s.null_count(), 1); assert_eq!(s.len(), 6); diff --git a/polars/polars-core/src/chunked_array/logical/struct_/mod.rs b/polars/polars-core/src/chunked_array/logical/struct_/mod.rs index c3070fb930f9..431f5c87a48f 100644 --- a/polars/polars-core/src/chunked_array/logical/struct_/mod.rs +++ b/polars/polars-core/src/chunked_array/logical/struct_/mod.rs @@ -23,7 +23,7 @@ fn fields_to_struct_array(fields: &[Series]) -> (ArrayRef, Vec) { let new_fields = fields.iter().map(|s| s.field().to_arrow()).collect(); let field_arrays = fields.iter().map(|s| s.to_arrow(0)).collect::>(); let arr = StructArray::new(ArrowDataType::Struct(new_fields), field_arrays, None); - (Arc::new(arr), fields) + (Box::new(arr), fields) } impl StructChunked { @@ -73,7 +73,7 @@ impl StructChunked { .iter() .map(|s| s.to_arrow(i)) .collect::>(); - let arr = Arc::new(StructArray::new( + let arr = Box::new(StructArray::new( ArrowDataType::Struct(new_fields.clone()), field_arrays, None, diff --git a/polars/polars-core/src/chunked_array/mod.rs b/polars/polars-core/src/chunked_array/mod.rs index 4cd71e2fd1be..c7354da2b276 100644 --- a/polars/polars-core/src/chunked_array/mod.rs +++ b/polars/polars-core/src/chunked_array/mod.rs @@ -229,8 +229,7 @@ impl ChunkedArray { .collect::>() .as_slice(), ) - .unwrap() - .into()]; + .unwrap()]; } /// Unpack a Series to the same physical type. @@ -350,7 +349,7 @@ impl ChunkedArray { .validity() .map(|bitmap| !bitmap) .unwrap_or_else(|| Bitmap::new_zeroed(arr.len())); - Arc::new(BooleanArray::from_data_default(bitmap, None)) as ArrayRef + Box::new(BooleanArray::from_data_default(bitmap, None)) as ArrayRef }) .collect::>(); BooleanChunked::from_chunks(self.name(), chunks) @@ -369,7 +368,7 @@ impl ChunkedArray { .validity() .cloned() .unwrap_or_else(|| !(&Bitmap::new_zeroed(arr.len()))); - Arc::new(BooleanArray::from_data_default(bitmap, None)) as ArrayRef + Box::new(BooleanArray::from_data_default(bitmap, None)) as ArrayRef }) .collect::>(); BooleanChunked::from_chunks(self.name(), chunks) @@ -416,7 +415,9 @@ where let mut offset = 0; let chunks = chunk_id .map(|len| { - let out = array.slice(offset, len).into(); + // safety: + // within bounds + let out = unsafe { array.slice_unchecked(offset, len) }; offset += len; out }) @@ -464,7 +465,7 @@ where // A hack to save compiler bloat for null arrays impl Int32Chunked { pub(crate) fn new_null(name: &str, len: usize) -> Self { - let arr = Arc::from(arrow::array::new_null_array(ArrowDataType::Null, len)); + let arr = arrow::array::new_null_array(ArrowDataType::Null, len); let field = Arc::new(Field::new(name, DataType::Null)); let chunks = vec![arr as ArrayRef]; ChunkedArray { @@ -635,12 +636,12 @@ pub(crate) fn to_array( values: Vec, validity: Option, ) -> ArrayRef { - Arc::new(to_primitive::(values, validity)) + Box::new(to_primitive::(values, validity)) } impl From> for ChunkedArray { fn from(a: PrimitiveArray) -> Self { - ChunkedArray::from_chunks("", vec![Arc::new(a)]) + ChunkedArray::from_chunks("", vec![Box::new(a)]) } } diff --git a/polars/polars-core/src/chunked_array/object/builder.rs b/polars/polars-core/src/chunked_array/object/builder.rs index 52aeddbaebf7..446be730ab17 100644 --- a/polars/polars-core/src/chunked_array/object/builder.rs +++ b/polars/polars-core/src/chunked_array/object/builder.rs @@ -62,7 +62,7 @@ where let len = self.values.len(); - let arr = Arc::new(ObjectArray { + let arr = Box::new(ObjectArray { values: Arc::new(self.values), null_bitmap, offset: 0, @@ -126,7 +126,7 @@ where let field = Arc::new(Field::new(name, DataType::Object(T::type_name()))); let len = v.len(); - let arr = Arc::new(ObjectArray { + let arr = Box::new(ObjectArray { values: Arc::new(v), null_bitmap: None, offset: 0, diff --git a/polars/polars-core/src/chunked_array/object/extension/drop.rs b/polars/polars-core/src/chunked_array/object/extension/drop.rs index 5dadb2f36809..b7ad89e55281 100644 --- a/polars/polars-core/src/chunked_array/object/extension/drop.rs +++ b/polars/polars-core/src/chunked_array/object/extension/drop.rs @@ -2,7 +2,7 @@ use crate::chunked_array::object::extension::PolarsExtension; use crate::prelude::*; /// This will dereference a raw ptr when dropping the PolarsExtension, make sure that it's valid. -pub(crate) unsafe fn drop_list(ca: &ListChunked) { +pub(crate) unsafe fn drop_list(ca: &mut ListChunked) { let mut inner = ca.inner_dtype(); let mut nested_count = 0; @@ -18,10 +18,7 @@ pub(crate) unsafe fn drop_list(ca: &ListChunked) { // if empty the memory is leaked somewhere assert!(!ca.chunks.is_empty()); for lst_arr in &ca.chunks { - // This list can be cloned, so we check the ref count before we drop - if let (ArrowDataType::LargeList(fld), 1) = - (lst_arr.data_type(), Arc::strong_count(lst_arr)) - { + if let ArrowDataType::LargeList(fld) = lst_arr.data_type() { let dtype = fld.data_type(); assert!(matches!(dtype, ArrowDataType::Extension(_, _, _))); @@ -31,15 +28,17 @@ pub(crate) unsafe fn drop_list(ca: &ListChunked) { let values = arr.values(); - // The inner value also may be cloned, check the ref count - if Arc::strong_count(values) == 1 { - let arr = values - .as_any() - .downcast_ref::() - .unwrap() - .clone(); - PolarsExtension::new(arr); - } + let arr = values + .as_any() + .downcast_ref::() + .unwrap(); + + // if the buf is not shared with anyone but us + // we can deallocate + let buf = arr.values(); + if buf.shared_count_strong() == 1 { + PolarsExtension::new(arr.clone()); + }; } } } diff --git a/polars/polars-core/src/chunked_array/object/extension/list.rs b/polars/polars-core/src/chunked_array/object/extension/list.rs index 91ace2949042..8eacb3c4b741 100644 --- a/polars/polars-core/src/chunked_array/object/extension/list.rs +++ b/polars/polars-core/src/chunked_array/object/extension/list.rs @@ -70,16 +70,20 @@ impl ListBuilderTrait for ExtensionListBuilder { // meaning that the sentinel is heap allocated and the dereference of the // pointer does not fail unsafe { pe.set_to_series_fn::() }; - let extension_array = Arc::new(pe.take_and_forget()) as ArrayRef; + let extension_array = Box::new(pe.take_and_forget()) as ArrayRef; let extension_dtype = extension_array.data_type(); let data_type = ListArray::::default_datatype(extension_dtype.clone()); - let arr = Arc::new(ListArray::::from_data( - data_type, - offsets.into(), - extension_array, - None, - )) as ArrayRef; + // Safety: + // offsets are monotonically increasing + let arr = unsafe { + Box::new(ListArray::::new_unchecked( + data_type, + offsets.into(), + extension_array, + None, + )) as ArrayRef + }; let mut listarr = ListChunked::from_chunks(ca.name(), vec![arr]); if self.fast_explode { diff --git a/polars/polars-core/src/chunked_array/object/mod.rs b/polars/polars-core/src/chunked_array/object/mod.rs index 25f718cc51e8..62a114f9da51 100644 --- a/polars/polars-core/src/chunked_array/object/mod.rs +++ b/polars/polars-core/src/chunked_array/object/mod.rs @@ -144,6 +144,10 @@ where fn to_boxed(&self) -> Box { Box::new(self.clone()) } + + fn as_any_mut(&mut self) -> &mut dyn Any { + unimplemented!() + } } impl ObjectChunked diff --git a/polars/polars-core/src/chunked_array/ops/any_value.rs b/polars/polars-core/src/chunked_array/ops/any_value.rs index f71421827a4c..3a169b4d25b3 100644 --- a/polars/polars-core/src/chunked_array/ops/any_value.rs +++ b/polars/polars-core/src/chunked_array/ops/any_value.rs @@ -41,7 +41,7 @@ pub(crate) unsafe fn arr_to_any_value<'a>( DataType::Float32 => downcast_and_pack!(Float32Array, Float32), DataType::Float64 => downcast_and_pack!(Float64Array, Float64), DataType::List(dt) => { - let v: ArrayRef = downcast!(LargeListArray).into(); + let v: ArrayRef = downcast!(LargeListArray); let mut s = Series::try_from(("", v)).unwrap(); match &**dt { diff --git a/polars/polars-core/src/chunked_array/ops/apply.rs b/polars/polars-core/src/chunked_array/ops/apply.rs index a308fb4e94b0..5fb591bfae7a 100644 --- a/polars/polars-core/src/chunked_array/ops/apply.rs +++ b/polars/polars-core/src/chunked_array/ops/apply.rs @@ -1,7 +1,7 @@ //! Implementations of the ChunkApply Trait. use crate::prelude::*; use crate::utils::{CustomIterTools, NoNull}; -use arrow::array::{ArrayRef, BooleanArray, PrimitiveArray}; +use arrow::array::{BooleanArray, PrimitiveArray}; use polars_arrow::array::PolarsArray; use polars_arrow::trusted_len::PushUnchecked; use std::borrow::Cow; @@ -69,7 +69,7 @@ where drop(arr); match owned_arr.into_mut() { - Left(immutable) => Arc::new(arrow::compute::arity::unary( + Left(immutable) => Box::new(arrow::compute::arity::unary( &immutable, f, S::get_dtype().to_arrow(), @@ -77,7 +77,8 @@ where Right(mut mutable) => { let vals = mutable.values_mut_slice(); vals.iter_mut().for_each(|v| *v = f(*v)); - mutable.into_arc() + let a: PrimitiveArray<_> = mutable.into(); + Box::new(a) as ArrayRef } } }) @@ -190,7 +191,7 @@ where let iter = arr.into_iter().map(|opt_v| f(opt_v.copied())); let arr = PrimitiveArray::::from_trusted_len_iter(iter) .to(T::get_dtype().to_arrow()); - Arc::new(arr) as ArrayRef + Box::new(arr) as ArrayRef }) .collect(); Self::from_chunks(self.name(), chunks) @@ -492,11 +493,8 @@ impl<'a> ChunkApply<'a, Series, Series> for ListChunked { .map(|array| { // safety // reported dtype is correct - let series = Series::from_chunks_and_dtype_unchecked( - "", - vec![array.into()], - &dtype, - ); + let series = + Series::from_chunks_and_dtype_unchecked("", vec![array], &dtype); f(series) }) .trust_my_length(self.len()) @@ -521,7 +519,6 @@ impl<'a> ChunkApply<'a, Series, Series> for ListChunked { .map(|array| { let values = array.iter().map(|x| { let x = x.map(|x| { - let x: ArrayRef = x.into(); // safety // reported dtype is correct unsafe { Series::from_chunks_and_dtype_unchecked("", vec![x], &dtype) } diff --git a/polars/polars-core/src/chunked_array/ops/bit_repr.rs b/polars/polars-core/src/chunked_array/ops/bit_repr.rs index 8931639c43f8..f454ae77f34f 100644 --- a/polars/polars-core/src/chunked_array/ops/bit_repr.rs +++ b/polars/polars-core/src/chunked_array/ops/bit_repr.rs @@ -1,5 +1,4 @@ use crate::prelude::*; -use arrow::array::Array; use arrow::buffer::Buffer; impl ToBitRepr for ChunkedArray @@ -28,11 +27,11 @@ where assert_eq!(reinterpretted_buf.len(), len); assert_eq!(reinterpretted_buf.offset(), offset); assert_eq!(reinterpretted_buf.as_slice().as_ptr() as usize, ptr); - Arc::new(PrimitiveArray::from_data( + Box::new(PrimitiveArray::from_data( ArrowDataType::UInt64, reinterpretted_buf, array.validity().cloned(), - )) as Arc + )) as ArrayRef }) .collect::>(); UInt64Chunked::from_chunks(self.name(), chunks) @@ -59,11 +58,11 @@ where assert_eq!(reinterpretted_buf.len(), len); assert_eq!(reinterpretted_buf.offset(), offset); assert_eq!(reinterpretted_buf.as_slice().as_ptr() as usize, ptr); - Arc::new(PrimitiveArray::from_data( + Box::new(PrimitiveArray::from_data( ArrowDataType::UInt32, reinterpretted_buf, array.validity().cloned(), - )) as Arc + )) as ArrayRef }) .collect::>(); UInt32Chunked::from_chunks(self.name(), chunks) @@ -92,11 +91,11 @@ impl Reinterpret for UInt64Chunked { assert_eq!(reinterpretted_buf.len(), len); assert_eq!(reinterpretted_buf.offset(), offset); assert_eq!(reinterpretted_buf.as_slice().as_ptr() as usize, ptr); - Arc::new(PrimitiveArray::from_data( + Box::new(PrimitiveArray::new( ArrowDataType::Int64, reinterpretted_buf, array.validity().cloned(), - )) as Arc + )) as ArrayRef }) .collect::>(); Int64Chunked::from_chunks(self.name(), chunks).into_series() @@ -135,11 +134,11 @@ impl UInt64Chunked { assert_eq!(reinterpretted_buf.len(), len); assert_eq!(reinterpretted_buf.offset(), offset); assert_eq!(reinterpretted_buf.as_slice().as_ptr() as usize, ptr); - Arc::new(PrimitiveArray::from_data( + Box::new(PrimitiveArray::from_data( ArrowDataType::Float64, reinterpretted_buf, array.validity().cloned(), - )) as Arc + )) as ArrayRef }) .collect::>(); Float64Chunked::from_chunks(self.name(), chunks).into() @@ -163,11 +162,11 @@ impl UInt32Chunked { assert_eq!(reinterpretted_buf.len(), len); assert_eq!(reinterpretted_buf.offset(), offset); assert_eq!(reinterpretted_buf.as_slice().as_ptr() as usize, ptr); - Arc::new(PrimitiveArray::from_data( + Box::new(PrimitiveArray::from_data( ArrowDataType::Float32, reinterpretted_buf, array.validity().cloned(), - )) as Arc + )) as ArrayRef }) .collect::>(); Float32Chunked::from_chunks(self.name(), chunks).into() diff --git a/polars/polars-core/src/chunked_array/ops/chunkops.rs b/polars/polars-core/src/chunked_array/ops/chunkops.rs index 5870f2343791..b573769227ea 100644 --- a/polars/polars-core/src/chunked_array/ops/chunkops.rs +++ b/polars/polars-core/src/chunked_array/ops/chunkops.rs @@ -35,7 +35,7 @@ fn slice( unsafe { // Safety: // this function ensures the slices are in bounds - new_chunks.push(chunk.slice_unchecked(remaining_offset, take_len).into()); + new_chunks.push(chunk.slice_unchecked(remaining_offset, take_len)); } remaining_length -= take_len; remaining_offset = 0; @@ -44,7 +44,7 @@ fn slice( } } if new_chunks.is_empty() { - new_chunks.push(chunks[0].slice(0, 0).into()); + new_chunks.push(chunks[0].slice(0, 0)); } new_chunks } @@ -64,8 +64,7 @@ where .collect::>() .as_slice(), ) - .unwrap() - .into()]; + .unwrap()]; ChunkedArray::from_chunks(self.name(), chunks) } } @@ -87,8 +86,7 @@ impl ChunkOps for BooleanChunked { .collect::>() .as_slice(), ) - .unwrap() - .into()]; + .unwrap()]; ChunkedArray::from_chunks(self.name(), chunks) } } @@ -110,8 +108,7 @@ impl ChunkOps for Utf8Chunked { .collect::>() .as_slice(), ) - .unwrap() - .into()]; + .unwrap()]; self.copy_with_chunks(chunks) } } @@ -133,8 +130,7 @@ impl ChunkOps for ListChunked { .collect::>() .as_slice(), ) - .unwrap() - .into()]; + .unwrap()]; let mut ca = ListChunked::from_chunks(self.name(), chunks); if self.can_fast_explode() { ca.set_fast_explode() diff --git a/polars/polars-core/src/chunked_array/ops/concat_str.rs b/polars/polars-core/src/chunked_array/ops/concat_str.rs index da72c08c2111..fe4ae128cf62 100644 --- a/polars/polars-core/src/chunked_array/ops/concat_str.rs +++ b/polars/polars-core/src/chunked_array/ops/concat_str.rs @@ -31,7 +31,7 @@ where let buf = buf.into_bytes(); let offsets = vec![0, buf.len() as i64]; let arr = unsafe { Utf8Array::from_data_unchecked_default(offsets.into(), buf.into(), None) }; - Utf8Chunked::from_chunks(name, vec![Arc::new(arr)]) + Utf8Chunked::from_chunks(name, vec![Box::new(arr)]) } impl StrConcat for ChunkedArray diff --git a/polars/polars-core/src/chunked_array/ops/explode.rs b/polars/polars-core/src/chunked_array/ops/explode.rs index 81faf76b9116..f3a3660c7332 100644 --- a/polars/polars-core/src/chunked_array/ops/explode.rs +++ b/polars/polars-core/src/chunked_array/ops/explode.rs @@ -111,7 +111,7 @@ where new_values.into(), Some(validity.into()), ); - Series::try_from((self.name(), Arc::new(arr) as ArrayRef)).unwrap() + Series::try_from((self.name(), Box::new(arr) as ArrayRef)).unwrap() } } @@ -254,7 +254,11 @@ impl ChunkExplode for ListChunked { if !offsets.is_empty() { let offset = offsets[0]; - values = Arc::from(values.slice(offset as usize, offsets[offsets.len() - 1] as usize)); + // safety: + // we are in bounds + values = unsafe { + values.slice_unchecked(offset as usize, offsets[offsets.len() - 1] as usize) + }; } let mut s = if ca.can_fast_explode() { @@ -356,7 +360,7 @@ impl ChunkExplode for Utf8Chunked { Utf8Array::::from_data_unchecked_default(offsets, values.clone(), validity) }; - let new_arr = Arc::new(array) as ArrayRef; + let new_arr = Box::new(array) as ArrayRef; let s = Series::try_from((self.name(), new_arr)).unwrap(); Ok((s, old_offsets)) diff --git a/polars/polars-core/src/chunked_array/ops/extend.rs b/polars/polars-core/src/chunked_array/ops/extend.rs index 5d01758623c4..0b10bddb9c93 100644 --- a/polars/polars-core/src/chunked_array/ops/extend.rs +++ b/polars/polars-core/src/chunked_array/ops/extend.rs @@ -11,7 +11,7 @@ fn extend_immutable(immutable: &dyn Array, chunks: &mut Vec, other_chu concatenate(&arrays).unwrap() }; - chunks.push(Arc::from(out)); + chunks.push(out); } impl ChunkedArray @@ -71,7 +71,7 @@ where } } let arr: PrimitiveArray = mutable.into(); - self.chunks.push(Arc::new(arr) as ArrayRef) + self.chunks.push(Box::new(arr) as ArrayRef) } } } @@ -107,7 +107,7 @@ impl Utf8Chunked { mutable.extend_trusted_len(arr.into_iter()) } let arr: Utf8Array = mutable.into(); - self.chunks.push(Arc::new(arr) as ArrayRef) + self.chunks.push(Box::new(arr) as ArrayRef) } } } @@ -144,7 +144,7 @@ impl BooleanChunked { mutable.extend_trusted_len(arr.into_iter()) } let arr: BooleanArray = mutable.into(); - self.chunks.push(Arc::new(arr) as ArrayRef) + self.chunks.push(Box::new(arr) as ArrayRef) } } } diff --git a/polars/polars-core/src/chunked_array/ops/fill_null.rs b/polars/polars-core/src/chunked_array/ops/fill_null.rs index 0351269d0e35..1f4f67da2a63 100644 --- a/polars/polars-core/src/chunked_array/ops/fill_null.rs +++ b/polars/polars-core/src/chunked_array/ops/fill_null.rs @@ -253,7 +253,7 @@ where T: PolarsNumericType, { fn fill_null_with_values(&self, value: T::Native) -> Result { - Ok(self.apply_kernel(&|arr| Arc::new(set_at_nulls(arr, value)))) + Ok(self.apply_kernel(&|arr| Box::new(set_at_nulls(arr, value)))) } } diff --git a/polars/polars-core/src/chunked_array/ops/filter.rs b/polars/polars-core/src/chunked_array/ops/filter.rs index bcc69b923eae..830f1aa65e02 100644 --- a/polars/polars-core/src/chunked_array/ops/filter.rs +++ b/polars/polars-core/src/chunked_array/ops/filter.rs @@ -43,7 +43,7 @@ where let chunks = left .downcast_iter() .zip(filter.downcast_iter()) - .map(|(left, mask)| filter_fn(left, mask).unwrap().into()) + .map(|(left, mask)| filter_fn(left, mask).unwrap()) .collect::>(); Ok(self.copy_with_chunks(chunks)) } @@ -64,7 +64,7 @@ impl ChunkFilter for BooleanChunked { let chunks = left .downcast_iter() .zip(filter.downcast_iter()) - .map(|(left, mask)| filter_fn(left, mask).unwrap().into()) + .map(|(left, mask)| filter_fn(left, mask).unwrap()) .collect::>(); Ok(self.copy_with_chunks(chunks)) } @@ -85,7 +85,7 @@ impl ChunkFilter for Utf8Chunked { let chunks = left .downcast_iter() .zip(filter.downcast_iter()) - .map(|(left, mask)| filter_fn(left, mask).unwrap().into()) + .map(|(left, mask)| filter_fn(left, mask).unwrap()) .collect::>(); Ok(self.copy_with_chunks(chunks)) @@ -106,7 +106,7 @@ impl ChunkFilter for ListChunked { let chunks = left .downcast_iter() .zip(filter.downcast_iter()) - .map(|(left, mask)| filter_fn(left, mask).unwrap().into()) + .map(|(left, mask)| filter_fn(left, mask).unwrap()) .collect::>(); // inner type may be categorical or logical type so we clone the state. diff --git a/polars/polars-core/src/chunked_array/ops/full.rs b/polars/polars-core/src/chunked_array/ops/full.rs index a8be315aca86..50cd84999b90 100644 --- a/polars/polars-core/src/chunked_array/ops/full.rs +++ b/polars/polars-core/src/chunked_array/ops/full.rs @@ -23,7 +23,7 @@ where T: PolarsNumericType, { fn full_null(name: &str, length: usize) -> Self { - let arr = new_null_array(T::get_dtype().to_arrow(), length).into(); + let arr = new_null_array(T::get_dtype().to_arrow(), length); ChunkedArray::from_chunks(name, vec![arr]) } } @@ -37,7 +37,7 @@ impl ChunkFull for BooleanChunked { impl ChunkFullNull for BooleanChunked { fn full_null(name: &str, length: usize) -> Self { - let arr = new_null_array(DataType::Boolean.to_arrow(), length).into(); + let arr = new_null_array(DataType::Boolean.to_arrow(), length); BooleanChunked::from_chunks(name, vec![arr]) } } @@ -55,7 +55,7 @@ impl<'a> ChunkFull<&'a str> for Utf8Chunked { impl ChunkFullNull for Utf8Chunked { fn full_null(name: &str, length: usize) -> Self { - let arr = new_null_array(DataType::Utf8.to_arrow(), length).into(); + let arr = new_null_array(DataType::Utf8.to_arrow(), length); Utf8Chunked::from_chunks(name, vec![arr]) } } @@ -86,8 +86,7 @@ impl ListChunked { true, ))), length, - ) - .into(); + ); ListChunked::from_chunks(name, vec![arr]) } } diff --git a/polars/polars-core/src/chunked_array/ops/interpolate.rs b/polars/polars-core/src/chunked_array/ops/interpolate.rs index f08e9e58a833..2d9b2ffd425a 100644 --- a/polars/polars-core/src/chunked_array/ops/interpolate.rs +++ b/polars/polars-core/src/chunked_array/ops/interpolate.rs @@ -141,7 +141,7 @@ impl ChunkedArray { av.into(), Some(validity.into()), ); - Self::from_chunks(self.name(), vec![Arc::new(array)]) + Self::from_chunks(self.name(), vec![Box::new(array)]) } else { Self::from_vec(self.name(), av) } diff --git a/polars/polars-core/src/chunked_array/ops/mod.rs b/polars/polars-core/src/chunked_array/ops/mod.rs index 872e146ade35..3e2c4ffe72ba 100644 --- a/polars/polars-core/src/chunked_array/ops/mod.rs +++ b/polars/polars-core/src/chunked_array/ops/mod.rs @@ -1,8 +1,6 @@ //! Traits for miscellaneous operations on ChunkedArray use std::marker::Sized; -use arrow::array::ArrayRef; - pub use self::take::*; #[cfg(feature = "object")] use crate::chunked_array::object::ObjectType; diff --git a/polars/polars-core/src/chunked_array/ops/repeat_by.rs b/polars/polars-core/src/chunked_array/ops/repeat_by.rs index 6d443f7e78bf..b4acc532e699 100644 --- a/polars/polars-core/src/chunked_array/ops/repeat_by.rs +++ b/polars/polars-core/src/chunked_array/ops/repeat_by.rs @@ -19,7 +19,7 @@ where // Length of iter is trusted ListChunked::from_chunks( self.name(), - vec![Arc::new(unsafe { + vec![Box::new(unsafe { LargeListArray::from_iter_primitive_trusted_len::( iter, T::get_dtype().to_arrow(), @@ -39,7 +39,7 @@ impl RepeatBy for BooleanChunked { // Length of iter is trusted ListChunked::from_chunks( self.name(), - vec![Arc::new(unsafe { + vec![Box::new(unsafe { LargeListArray::from_iter_bool_trusted_len(iter) })], ) @@ -56,7 +56,7 @@ impl RepeatBy for Utf8Chunked { // Length of iter is trusted ListChunked::from_chunks( self.name(), - vec![Arc::new(unsafe { + vec![Box::new(unsafe { LargeListArray::from_iter_utf8_trusted_len(iter, self.len()) })], ) diff --git a/polars/polars-core/src/chunked_array/ops/rolling_window.rs b/polars/polars-core/src/chunked_array/ops/rolling_window.rs index 3b28a4d32218..5b90802fa08e 100644 --- a/polars/polars-core/src/chunked_array/ops/rolling_window.rs +++ b/polars/polars-core/src/chunked_array/ops/rolling_window.rs @@ -88,8 +88,9 @@ mod inner_mod { let arr = ca.downcast_iter().next().unwrap(); let series_container = ChunkedArray::::from_slice("", &[T::Native::zero()]).into_series(); - let array_ptr = &series_container.array_ref(0); - let ptr = Arc::as_ptr(array_ptr) as *mut dyn Array as *mut PrimitiveArray; + let array_ptr = series_container.array_ref(0); + let ptr = &*array_ptr.as_ref() as *const dyn Array as *mut dyn Array + as *mut PrimitiveArray; let mut builder = PrimitiveChunkedBuilder::::new(self.name(), self.len()); if let Some(weights) = options.weights { @@ -186,7 +187,8 @@ mod inner_mod { let arr_container = ChunkedArray::::from_slice("", &[T::Native::zero()]); let array_ptr = &arr_container.chunks()[0]; - let ptr = Arc::as_ptr(array_ptr) as *mut dyn Array as *mut PrimitiveArray; + let ptr = &*array_ptr.as_ref() as *const dyn Array as *mut dyn Array + as *mut PrimitiveArray; let mut validity = MutableBitmap::with_capacity(ca.len()); validity.extend_constant(window_size - 1, false); @@ -218,7 +220,7 @@ mod inner_mod { values.into(), Some(validity.into()), ); - Ok(Self::from_chunks(self.name(), vec![Arc::new(arr)])) + Ok(Self::from_chunks(self.name(), vec![Box::new(arr)])) } } } diff --git a/polars/polars-core/src/chunked_array/ops/set.rs b/polars/polars-core/src/chunked_array/ops/set.rs index 34581a75c4bb..0af6461425c2 100644 --- a/polars/polars-core/src/chunked_array/ops/set.rs +++ b/polars/polars-core/src/chunked_array/ops/set.rs @@ -1,11 +1,9 @@ use crate::prelude::*; use crate::utils::{align_chunks_binary, CustomIterTools}; -use arrow::array::ArrayRef; use arrow::bitmap::MutableBitmap; use polars_arrow::array::ValueSize; use polars_arrow::kernels::set::{set_at_idx_no_null, set_with_mask}; use polars_arrow::prelude::FromData; -use std::sync::Arc; macro_rules! impl_set_at_idx_with { ($self:ident, $builder:ident, $idx:ident, $f:ident) => {{ @@ -71,7 +69,7 @@ where value, T::get_dtype().to_arrow(), )?; - return Ok(Self::from_chunks(self.name(), vec![Arc::new(arr)])); + return Ok(Self::from_chunks(self.name(), vec![Box::new(arr)])); } // Other fast path. Slightly slower as it does not do a memcpy else { @@ -117,7 +115,7 @@ where .zip(mask.downcast_iter()) .map(|(arr, mask)| { let a = set_with_mask(arr, mask, value, T::get_dtype().to_arrow()); - Arc::new(a) as ArrayRef + Box::new(a) as ArrayRef }) .collect(); Ok(ChunkedArray::from_chunks(self.name(), chunks)) @@ -192,7 +190,7 @@ impl<'a> ChunkSet<'a, bool, bool> for BooleanChunked { Ok(BooleanChunked::from_chunks( self.name(), - vec![Arc::new(arr)], + vec![Box::new(arr)], )) } diff --git a/polars/polars-core/src/chunked_array/ops/sort/argsort.rs b/polars/polars-core/src/chunked_array/ops/sort/argsort.rs index 8dfef645e662..482a69c6f55a 100644 --- a/polars/polars-core/src/chunked_array/ops/sort/argsort.rs +++ b/polars/polars-core/src/chunked_array/ops/sort/argsort.rs @@ -71,5 +71,5 @@ where }; let arr = IdxArr::from_data_default(Buffer::from(idx), None); - IdxCa::from_chunks(name, vec![Arc::new(arr)]) + IdxCa::from_chunks(name, vec![Box::new(arr)]) } diff --git a/polars/polars-core/src/chunked_array/ops/sort/categorical.rs b/polars/polars-core/src/chunked_array/ops/sort/categorical.rs index fb540610512f..e6c5400051fa 100644 --- a/polars/polars-core/src/chunked_array/ops/sort/categorical.rs +++ b/polars/polars-core/src/chunked_array/ops/sort/categorical.rs @@ -34,7 +34,7 @@ impl CategoricalChunked { RevMapping::Local(arr) => { // we don't use arrow2 sort here because its not activated // that saves compilation - let ca = Utf8Chunked::from_chunks("", vec![Arc::from(arr.clone())]); + let ca = Utf8Chunked::from_chunks("", vec![Box::from(arr.clone())]); let sorted = ca.sort(options.descending); let arr = sorted.downcast_iter().next().unwrap().clone(); let rev_map = RevMapping::Local(arr); diff --git a/polars/polars-core/src/chunked_array/ops/take/mod.rs b/polars/polars-core/src/chunked_array/ops/take/mod.rs index e6f4a0215007..d7450d65937a 100644 --- a/polars/polars-core/src/chunked_array/ops/take/mod.rs +++ b/polars/polars-core/src/chunked_array/ops/take/mod.rs @@ -4,7 +4,6 @@ //! There are several structs that implement the fastest path for random access. //! -use arrow::array::ArrayRef; use arrow::compute::take::take; use polars_arrow::kernels::take::*; @@ -176,7 +175,7 @@ impl ChunkTake for BooleanChunked { return Self::full_null(self.name(), array.len()); } let array = match self.chunks.len() { - 1 => take::take(chunks.next().unwrap(), array).unwrap().into(), + 1 => take::take(chunks.next().unwrap(), array).unwrap(), _ => { return if !array.has_validity() { let iter = array.values().iter().map(|i| *i as usize); @@ -351,7 +350,7 @@ impl ChunkTake for ListChunked { ); } let array = match ca_self.chunks.len() { - 1 => Arc::new(take_list_unchecked(chunks.next().unwrap(), array)) as ArrayRef, + 1 => Box::new(take_list_unchecked(chunks.next().unwrap(), array)) as ArrayRef, _ => { return if !array.has_validity() { let iter = array.values().iter().map(|i| *i as usize); diff --git a/polars/polars-core/src/chunked_array/ops/take/take_random.rs b/polars/polars-core/src/chunked_array/ops/take/take_random.rs index 2d18a89c07e2..70a63afa92e0 100644 --- a/polars/polars-core/src/chunked_array/ops/take/take_random.rs +++ b/polars/polars-core/src/chunked_array/ops/take/take_random.rs @@ -429,7 +429,6 @@ impl<'a> TakeRandom for ListTakeRandom<'a> { fn get(&self, index: usize) -> Option { let v = take_random_get!(self, index); v.map(|v| { - let v: Arc = v.into(); let s = Series::try_from((self.ca.name(), v)); s.unwrap() }) diff --git a/polars/polars-core/src/chunked_array/ops/take/take_single.rs b/polars/polars-core/src/chunked_array/ops/take/take_single.rs index b69f29b197bc..3390b1ec30dc 100644 --- a/polars/polars-core/src/chunked_array/ops/take/take_single.rs +++ b/polars/polars-core/src/chunked_array/ops/take/take_single.rs @@ -4,7 +4,6 @@ use crate::prelude::*; use arrow::array::*; use polars_arrow::is_valid::IsValid; use std::convert::TryFrom; -use std::sync::Arc; macro_rules! impl_take_random_get { ($self:ident, $index:ident, $array_type:ty) => {{ @@ -16,7 +15,7 @@ macro_rules! impl_take_random_get { // Safety: // caller should give right array type - let arr = &*(arr as *const ArrayRef as *const Arc<$array_type>); + let arr = &*(arr as *const ArrayRef as *const Box<$array_type>); // Safety: // index should be in bounds @@ -37,7 +36,7 @@ macro_rules! impl_take_random_get_unchecked { // Safety: // caller should give right array type - let arr = &*(arr as *const ArrayRef as *const Arc<$array_type>); + let arr = &*(&**arr as *const dyn Array as *const $array_type); // Safety: // index should be in bounds diff --git a/polars/polars-core/src/chunked_array/ops/unique/mod.rs b/polars/polars-core/src/chunked_array/ops/unique/mod.rs index dc0b8815ad40..4a906e537dd0 100644 --- a/polars/polars-core/src/chunked_array/ops/unique/mod.rs +++ b/polars/polars-core/src/chunked_array/ops/unique/mod.rs @@ -314,7 +314,7 @@ impl ChunkUnique for Float64Chunked { mod is_first { use super::*; use crate::utils::CustomIterTools; - use arrow::array::{ArrayRef, BooleanArray}; + use arrow::array::BooleanArray; fn is_first(ca: &ChunkedArray) -> BooleanChunked where @@ -329,7 +329,7 @@ mod is_first { .into_iter() .map(|opt_v| unique.insert(opt_v)) .collect_trusted(); - Arc::new(mask) as ArrayRef + Box::new(mask) as ArrayRef }) .collect(); @@ -371,7 +371,7 @@ mod is_first { .into_iter() .map(|opt_v| unique.insert(opt_v)) .collect_trusted(); - Arc::new(mask) as ArrayRef + Box::new(mask) as ArrayRef }) .collect(); diff --git a/polars/polars-core/src/chunked_array/ops/zip.rs b/polars/polars-core/src/chunked_array/ops/zip.rs index f1f2a8623451..97061188abb2 100644 --- a/polars/polars-core/src/chunked_array/ops/zip.rs +++ b/polars/polars-core/src/chunked_array/ops/zip.rs @@ -82,7 +82,7 @@ where .zip(mask.downcast_iter()) .map(|((left_c, right_c), mask_c)| { let mask_c = prepare_mask(mask_c); - let arr = if_then_else(&mask_c, left_c, right_c)?.into(); + let arr = if_then_else(&mask_c, left_c, right_c)?; Ok(arr) }) .collect::>>()?; @@ -104,7 +104,7 @@ impl ChunkZip for BooleanChunked { .zip(mask.downcast_iter()) .map(|((left_c, right_c), mask_c)| { let mask_c = prepare_mask(mask_c); - let arr = if_then_else(&mask_c, left_c, right_c)?.into(); + let arr = if_then_else(&mask_c, left_c, right_c)?; Ok(arr) }) .collect::>>()?; @@ -125,7 +125,7 @@ impl ChunkZip for Utf8Chunked { .zip(mask.downcast_iter()) .map(|((left_c, right_c), mask_c)| { let mask_c = prepare_mask(mask_c); - let arr = if_then_else(&mask_c, left_c, right_c)?.into(); + let arr = if_then_else(&mask_c, left_c, right_c)?; Ok(arr) }) .collect::>>()?; @@ -146,7 +146,7 @@ impl ChunkZip for ListChunked { .zip(mask.downcast_iter()) .map(|((left_c, right_c), mask_c)| { let mask_c = prepare_mask(mask_c); - let arr = if_then_else(&mask_c, left_c, right_c)?.into(); + let arr = if_then_else(&mask_c, left_c, right_c)?; Ok(arr) }) .collect::>>()?; diff --git a/polars/polars-core/src/chunked_array/temporal/conversion.rs b/polars/polars-core/src/chunked_array/temporal/conversion.rs index 8ce26a205525..c8da40d5732d 100644 --- a/polars/polars-core/src/chunked_array/temporal/conversion.rs +++ b/polars/polars-core/src/chunked_array/temporal/conversion.rs @@ -1,10 +1,8 @@ -use crate::prelude::{AnyValue, TimeUnit}; -#[cfg(feature = "dtype-time")] -use arrow::temporal_conversions::time64ns_to_time; -use arrow::temporal_conversions::{ - timestamp_ms_to_datetime, timestamp_ns_to_datetime, timestamp_us_to_datetime, MILLISECONDS, -}; -use chrono::{NaiveDateTime, NaiveTime}; +use crate::prelude::*; + +use arrow::temporal_conversions::*; + +use chrono::*; /// Number of seconds in a day pub(crate) const SECONDS_IN_DAY: i64 = 86_400; diff --git a/polars/polars-core/src/chunked_array/temporal/date.rs b/polars/polars-core/src/chunked_array/temporal/date.rs index 82d9a6aad616..4d1bc3cc2aa2 100644 --- a/polars/polars-core/src/chunked_array/temporal/date.rs +++ b/polars/polars-core/src/chunked_array/temporal/date.rs @@ -51,7 +51,7 @@ impl DateChunked { } let arr: Utf8Array = mutarr.into(); - Arc::new(arr) + Box::new(arr) }); ca.rename(self.name()); ca diff --git a/polars/polars-core/src/chunked_array/temporal/datetime.rs b/polars/polars-core/src/chunked_array/temporal/datetime.rs index 4d73119893be..44cc4b248c2a 100644 --- a/polars/polars-core/src/chunked_array/temporal/datetime.rs +++ b/polars/polars-core/src/chunked_array/temporal/datetime.rs @@ -67,7 +67,7 @@ impl DatetimeChunked { } let arr: Utf8Array = mutarr.into(); - Arc::new(arr) + Box::new(arr) }); ca.rename(self.name()); ca diff --git a/polars/polars-core/src/chunked_array/temporal/mod.rs b/polars/polars-core/src/chunked_array/temporal/mod.rs index 8b7365c268da..ad5561cc065f 100644 --- a/polars/polars-core/src/chunked_array/temporal/mod.rs +++ b/polars/polars-core/src/chunked_array/temporal/mod.rs @@ -8,10 +8,15 @@ mod datetime; mod duration; #[cfg(feature = "dtype-time")] mod time; -mod utf8; pub use self::conversion::*; -use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; + +use chrono::NaiveDateTime; +#[cfg(feature = "dtype-time")] +use chrono::NaiveTime; + +#[cfg(feature = "dtype-date")] +use chrono::NaiveDate; pub fn unix_time() -> NaiveDateTime { NaiveDateTime::from_timestamp(0, 0) diff --git a/polars/polars-core/src/chunked_array/temporal/utf8.rs b/polars/polars-core/src/chunked_array/temporal/utf8.rs deleted file mode 100644 index 232727836a1c..000000000000 --- a/polars/polars-core/src/chunked_array/temporal/utf8.rs +++ /dev/null @@ -1,355 +0,0 @@ -// use super::*; -// #[cfg(feature = "dtype-time")] -// use crate::chunked_array::temporal::time::time_to_time64ns; -// use crate::prelude::*; -// use chrono; -// use chrono::ParseError; -// -// #[cfg(feature = "dtype-time")] -// fn time_pattern(val: &str, convert: F) -> Option<&'static str> -// // (string, fmt) -> result -// where -// F: Fn(&str, &str) -> chrono::ParseResult, -// { -// for fmt in ["%T", "%T%.3f", "%T%.6f", "%T%.9f"] { -// if convert(val, fmt).is_ok() { -// return Some(fmt); -// } -// } -// None -// } -// -// fn datetime_pattern(val: &str, convert: F) -> Option<&'static str> -// // (string, fmt) -> result -// where -// F: Fn(&str, &str) -> chrono::ParseResult, -// { -// for fmt in [ -// // 21/12/31 12:54:98 -// "%y/%m/%d %H:%M:%S", -// // 2021-12-31 24:58:01 -// "%y-%m-%d %H:%M:%S", -// // 21/12/31 24:58:01 -// "%y/%m/%d %H:%M:%S", -// //210319 23:58:50 -// "%y%m%d %H:%M:%S", -// // 2019-04-18T02:45:55 -// // 2021/12/31 12:54:98 -// "%Y/%m/%d %H:%M:%S", -// // 2021-12-31 24:58:01 -// "%Y-%m-%d %H:%M:%S", -// // 2021/12/31 24:58:01 -// "%Y/%m/%d %H:%M:%S", -// // 20210319 23:58:50 -// "%Y%m%d %H:%M:%S", -// // 2019-04-18T02:45:55 -// "%FT%H:%M:%S", -// // 2019-04-18T02:45:55.555000000 -// // microseconds -// "%FT%H:%M:%S.%6f", -// // nanoseconds -// "%FT%H:%M:%S.%9f", -// ] { -// if convert(val, fmt).is_ok() { -// return Some(fmt); -// } -// } -// None -// } -// -// fn date_pattern(val: &str, convert: F) -> Option<&'static str> -// // (string, fmt) -> result -// where -// F: Fn(&str, &str) -> chrono::ParseResult, -// { -// for fmt in [ -// // 2021-12-31 -// "%Y-%m-%d", // 31-12-2021 -// "%d-%m-%Y", // 2021319 (2021-03-19) -// "%Y%m%d", -// ] { -// if convert(val, fmt).is_ok() { -// return Some(fmt); -// } -// } -// None -// } -// -// struct ParseErrorByteCopy(ParseErrorKind); -// -// impl From for ParseErrorByteCopy { -// fn from(e: ParseError) -> Self { -// // we need to do this until chrono ParseErrorKind is public -// // blocked by https://github.com/chronotope/chrono/pull/588 -// unsafe { std::mem::transmute(e) } -// } -// } -// -// #[allow(dead_code)] -// enum ParseErrorKind { -// OutOfRange, -// Impossible, -// NotEnough, -// Invalid, -// /// The input string has been prematurely ended. -// TooShort, -// TooLong, -// BadFormat, -// } -// -// impl Utf8Chunked { -// fn get_first_val(&self) -> Result<&str> { -// let idx = match self.first_non_null() { -// Some(idx) => idx, -// None => { -// return Err(PolarsError::ComputeError( -// "Cannot determine date parsing format, all values are null".into(), -// )) -// } -// }; -// let val = self.get(idx).expect("should not be null"); -// Ok(val) -// } -// -// #[cfg(feature = "dtype-datetime")] -// fn sniff_fmt_datetime(&self) -> Result<&'static str> { -// let val = self.get_first_val()?; -// if let Some(pattern) = datetime_pattern(val, NaiveDateTime::parse_from_str) { -// return Ok(pattern); -// } -// Err(PolarsError::ComputeError( -// "Could not find an appropriate format to parse dates, please define a fmt".into(), -// )) -// } -// -// #[cfg(feature = "dtype-date")] -// fn sniff_fmt_date(&self) -> Result<&'static str> { -// let val = self.get_first_val()?; -// if let Some(pattern) = date_pattern(val, NaiveDate::parse_from_str) { -// return Ok(pattern); -// } -// Err(PolarsError::ComputeError( -// "Could not find an appropriate format to parse dates, please define a fmt".into(), -// )) -// } -// -// #[cfg(feature = "dtype-time")] -// fn sniff_fmt_time(&self) -> Result<&'static str> { -// let val = self.get_first_val()?; -// if let Some(pattern) = time_pattern(val, NaiveTime::parse_from_str) { -// return Ok(pattern); -// } -// Err(PolarsError::ComputeError( -// "Could not find an appropriate format to parse times, please define a fmt".into(), -// )) -// } -// -// #[cfg(feature = "dtype-time")] -// /// Parsing string values and return a [`TimeChunked`] -// pub fn as_time(&self, fmt: Option<&str>) -> Result { -// let fmt = match fmt { -// Some(fmt) => fmt, -// None => self.sniff_fmt_time()?, -// }; -// -// let mut ca: Int64Chunked = match self.has_validity() { -// false => self -// .into_no_null_iter() -// .map(|s| { -// NaiveTime::parse_from_str(s, fmt) -// .ok() -// .as_ref() -// .map(time_to_time64ns) -// }) -// .collect_trusted(), -// _ => self -// .into_iter() -// .map(|opt_s| { -// let opt_nd = opt_s.map(|s| { -// NaiveTime::parse_from_str(s, fmt) -// .ok() -// .as_ref() -// .map(time_to_time64ns) -// }); -// match opt_nd { -// None => None, -// Some(None) => None, -// Some(Some(nd)) => Some(nd), -// } -// }) -// .collect_trusted(), -// }; -// ca.rename(self.name()); -// Ok(ca.into()) -// } -// -// #[cfg(feature = "dtype-date")] -// /// Parsing string values and return a [`DateChunked`] -// /// Different from `as_date` this function allows matches that not contain the whole string -// /// e.g. "foo-2021-01-01-bar" could match "2021-01-01" -// pub fn as_date_not_exact(&self, fmt: Option<&str>) -> Result { -// let fmt = match fmt { -// Some(fmt) => fmt, -// None => self.sniff_fmt_date()?, -// }; -// let mut ca: Int32Chunked = self -// .into_iter() -// .map(|opt_s| match opt_s { -// None => None, -// Some(mut s) => { -// let fmt_len = fmt.len(); -// -// for i in 1..(s.len() - fmt_len) { -// if s.is_empty() { -// return None; -// } -// match NaiveDate::parse_from_str(s, fmt).map(naive_date_to_date) { -// Ok(nd) => return Some(nd), -// Err(e) => { -// let e: ParseErrorByteCopy = e.into(); -// match e.0 { -// ParseErrorKind::TooLong => { -// s = &s[..s.len() - 1]; -// } -// _ => { -// s = &s[i..]; -// } -// } -// } -// } -// } -// None -// } -// }) -// .collect_trusted(); -// ca.rename(self.name()); -// Ok(ca.into()) -// } -// -// #[cfg(feature = "dtype-datetime")] -// /// Parsing string values and return a [`DatetimeChunked`] -// /// Different from `as_datetime` this function allows matches that not contain the whole string -// /// e.g. "foo-2021-01-01-bar" could match "2021-01-01" -// pub fn as_datetime_not_exact( -// &self, -// fmt: Option<&str>, -// tu: TimeUnit, -// ) -> Result { -// let fmt = match fmt { -// Some(fmt) => fmt, -// None => self.sniff_fmt_datetime()?, -// }; -// -// let func = match tu { -// TimeUnit::Nanoseconds => datetime_to_timestamp_ns, -// TimeUnit::Microseconds => datetime_to_timestamp_us, -// TimeUnit::Milliseconds => datetime_to_timestamp_ms, -// }; -// -// let mut ca: Int64Chunked = self -// .into_iter() -// .map(|opt_s| match opt_s { -// None => None, -// Some(mut s) => { -// let fmt_len = fmt.len(); -// -// for i in 1..(s.len() - fmt_len) { -// if s.is_empty() { -// return None; -// } -// match NaiveDateTime::parse_from_str(s, fmt).map(func) { -// Ok(nd) => return Some(nd), -// Err(e) => { -// let e: ParseErrorByteCopy = e.into(); -// match e.0 { -// ParseErrorKind::TooLong => { -// s = &s[..s.len() - 1]; -// } -// _ => { -// s = &s[i..]; -// } -// } -// } -// } -// } -// None -// } -// }) -// .collect_trusted(); -// ca.rename(self.name()); -// Ok(ca.into_datetime(tu, None)) -// } -// -// #[cfg(feature = "dtype-date")] -// /// Parsing string values and return a [`DateChunked`] -// pub fn as_date(&self, fmt: Option<&str>) -> Result { -// let fmt = match fmt { -// Some(fmt) => fmt, -// None => self.sniff_fmt_date()?, -// }; -// -// let mut ca: Int32Chunked = match self.has_validity() { -// false => self -// .into_no_null_iter() -// .map(|s| { -// NaiveDate::parse_from_str(s, fmt) -// .ok() -// .map(naive_date_to_date) -// }) -// .collect_trusted(), -// _ => self -// .into_iter() -// .map(|opt_s| { -// let opt_nd = opt_s.map(|s| { -// NaiveDate::parse_from_str(s, fmt) -// .ok() -// .map(naive_date_to_date) -// }); -// match opt_nd { -// None => None, -// Some(None) => None, -// Some(Some(nd)) => Some(nd), -// } -// }) -// .collect_trusted(), -// }; -// ca.rename(self.name()); -// Ok(ca.into()) -// } -// -// #[cfg(feature = "dtype-datetime")] -// /// Parsing string values and return a [`DatetimeChunked`] -// pub fn as_datetime(&self, fmt: Option<&str>, tu: TimeUnit) -> Result { -// let fmt = match fmt { -// Some(fmt) => fmt, -// None => self.sniff_fmt_datetime()?, -// }; -// -// let func = match tu { -// TimeUnit::Nanoseconds => datetime_to_timestamp_ns, -// TimeUnit::Microseconds => datetime_to_timestamp_us, -// TimeUnit::Milliseconds => datetime_to_timestamp_ms, -// }; -// -// let mut ca: Int64Chunked = match self.has_validity() { -// false => self -// .into_no_null_iter() -// .map(|s| NaiveDateTime::parse_from_str(s, fmt).ok().map(func)) -// .collect_trusted(), -// _ => self -// .into_iter() -// .map(|opt_s| { -// let opt_nd = -// opt_s.map(|s| NaiveDateTime::parse_from_str(s, fmt).ok().map(func)); -// match opt_nd { -// None => None, -// Some(None) => None, -// Some(Some(nd)) => Some(nd), -// } -// }) -// .collect_trusted(), -// }; -// ca.rename(self.name()); -// Ok(ca.into_datetime(tu, None)) -// } -// } diff --git a/polars/polars-core/src/chunked_array/trusted_len.rs b/polars/polars-core/src/chunked_array/trusted_len.rs index a090809587e7..4bc846170966 100644 --- a/polars/polars-core/src/chunked_array/trusted_len.rs +++ b/polars/polars-core/src/chunked_array/trusted_len.rs @@ -16,7 +16,7 @@ where let arr = unsafe { PrimitiveArray::from_trusted_len_iter_unchecked(iter).to(T::get_dtype().to_arrow()) }; - ChunkedArray::from_chunks("", vec![Arc::new(arr)]) + ChunkedArray::from_chunks("", vec![Box::new(arr)]) } } @@ -32,7 +32,7 @@ where let values = unsafe { Vec::from_trusted_len_iter_unchecked(iter) }.into(); let arr = PrimitiveArray::from_data(T::get_dtype().to_arrow(), values, None); - NoNull::new(ChunkedArray::from_chunks("", vec![Arc::new(arr)])) + NoNull::new(ChunkedArray::from_chunks("", vec![Box::new(arr)])) } } @@ -72,7 +72,7 @@ where vals.into(), Some(validity.into()), ); - ChunkedArray::from_chunks("", vec![Arc::new(arr)]) + ChunkedArray::from_chunks("", vec![Box::new(arr)]) } } @@ -107,7 +107,7 @@ impl FromIteratorReversed> for BooleanChunked { } let arr = BooleanArray::from_data(ArrowDataType::Boolean, vals.into(), Some(validity.into())); - ChunkedArray::from_chunks("", vec![Arc::new(arr)]) + ChunkedArray::from_chunks("", vec![Box::new(arr)]) } } @@ -130,7 +130,7 @@ where vals.set_len(size) } let arr = PrimitiveArray::from_data(T::get_dtype().to_arrow(), vals.into(), None); - NoNull::new(ChunkedArray::from_chunks("", vec![Arc::new(arr)])) + NoNull::new(ChunkedArray::from_chunks("", vec![Box::new(arr)])) } } @@ -159,7 +159,7 @@ impl FromTrustedLenIterator> for ChunkedArray { let iter = iter.into_iter(); let arr: BooleanArray = iter.collect_trusted(); - Self::from_chunks("", vec![Arc::new(arr)]) + Self::from_chunks("", vec![Box::new(arr)]) } } @@ -171,7 +171,7 @@ impl FromTrustedLenIterator for BooleanChunked { let iter = iter.into_iter(); let arr: BooleanArray = iter.collect_trusted(); - Self::from_chunks("", vec![Arc::new(arr)]) + Self::from_chunks("", vec![Box::new(arr)]) } } diff --git a/polars/polars-core/src/chunked_array/upstream_traits.rs b/polars/polars-core/src/chunked_array/upstream_traits.rs index b485b488cb5f..0f6843c08b07 100644 --- a/polars/polars-core/src/chunked_array/upstream_traits.rs +++ b/polars/polars-core/src/chunked_array/upstream_traits.rs @@ -60,7 +60,7 @@ where .collect::>() .to(T::get_dtype().to_arrow()), }; - ChunkedArray::from_chunks("", vec![Arc::new(arr)]) + ChunkedArray::from_chunks("", vec![Box::new(arr)]) } } @@ -81,7 +81,7 @@ where impl FromIterator> for ChunkedArray { fn from_iter>>(iter: I) -> Self { let arr = BooleanArray::from_iter(iter); - Self::from_chunks("", vec![Arc::new(arr)]) + Self::from_chunks("", vec![Box::new(arr)]) } } @@ -89,7 +89,7 @@ impl FromIterator for BooleanChunked { fn from_iter>(iter: I) -> Self { // 2021-02-07: this was ~70% faster than with the builder, even with the extra Option added. let arr = BooleanArray::from_iter(iter.into_iter().map(Some)); - Self::from_chunks("", vec![Arc::new(arr)]) + Self::from_chunks("", vec![Box::new(arr)]) } } @@ -108,7 +108,7 @@ where { fn from_iter>>(iter: I) -> Self { let arr = Utf8Array::::from_iter(iter); - Self::from_chunks("", vec![Arc::new(arr)]) + Self::from_chunks("", vec![Box::new(arr)]) } } @@ -127,7 +127,7 @@ where { fn from_iter>(iter: I) -> Self { let arr = Utf8Array::::from_iter_values(iter.into_iter()); - Self::from_chunks("", vec![Arc::new(arr)]) + Self::from_chunks("", vec![Box::new(arr)]) } } @@ -295,7 +295,7 @@ impl FromIterator> for ObjectChunked { let len = values.len(); - let arr = Arc::new(ObjectArray { + let arr = Box::new(ObjectArray { values: Arc::new(values), null_bitmap, offset: 0, @@ -377,7 +377,7 @@ where let iter = TrustMyLength::new(vectors.into_iter().flatten(), capacity); let arr = PrimitiveArray::::from_trusted_len_iter(iter).to(T::get_dtype().to_arrow()); - Self::from_chunks("", vec![Arc::new(arr)]) + Self::from_chunks("", vec![Box::new(arr)]) } } @@ -392,7 +392,7 @@ impl FromParallelIterator for BooleanChunked { vectors.into_iter().flatten().trust_my_length(capacity), ) }; - Self::from_chunks("", vec![Arc::new(arr)]) + Self::from_chunks("", vec![Box::new(arr)]) } } @@ -403,7 +403,7 @@ where fn from_par_iter>(iter: I) -> Self { let vectors = collect_into_linked_list(iter); let arr = LargeStringArray::from_iter_values(vectors.into_iter().flatten()); - Self::from_chunks("", vec![Arc::new(arr)]) + Self::from_chunks("", vec![Box::new(arr)]) } } @@ -414,7 +414,7 @@ where fn from_par_iter>>(iter: I) -> Self { let vectors = collect_into_linked_list(iter); let arr = LargeStringArray::from_iter(vectors.into_iter().flatten()); - Self::from_chunks("", vec![Arc::new(arr)]) + Self::from_chunks("", vec![Box::new(arr)]) } } diff --git a/polars/polars-core/src/datatypes.rs b/polars/polars-core/src/datatypes.rs index 204e98acbb53..f910af283896 100644 --- a/polars/polars-core/src/datatypes.rs +++ b/polars/polars-core/src/datatypes.rs @@ -413,18 +413,18 @@ impl<'a> AnyValue<'a> { dt => panic!("cannot create date from other type. dtype: {}", dt), } } + #[cfg(feature = "dtype-datetime")] pub(crate) fn into_datetime(self, tu: TimeUnit, tz: &'a Option) -> Self { match self { - #[cfg(feature = "dtype-datetime")] AnyValue::Int64(v) => AnyValue::Datetime(v, tu, tz), AnyValue::Null => AnyValue::Null, dt => panic!("cannot create date from other type. dtype: {}", dt), } } + #[cfg(feature = "dtype-duration")] pub(crate) fn into_duration(self, tu: TimeUnit) -> Self { match self { - #[cfg(feature = "dtype-duration")] AnyValue::Int64(v) => AnyValue::Duration(v, tu), AnyValue::Null => AnyValue::Null, dt => panic!("cannot create date from other type. dtype: {}", dt), diff --git a/polars/polars-core/src/fmt.rs b/polars/polars-core/src/fmt.rs index b591ff01101c..f7b5230b9e03 100644 --- a/polars/polars-core/src/fmt.rs +++ b/polars/polars-core/src/fmt.rs @@ -1,7 +1,5 @@ use crate::prelude::*; -#[cfg(any(feature = "dtype-date", feature = "dtype-datetime"))] -use arrow::temporal_conversions::{date32_to_date, timestamp_ns_to_datetime}; #[cfg(feature = "timezones")] use chrono::TimeZone; use num::{Num, NumCast}; @@ -12,7 +10,12 @@ use std::{ const LIMIT: usize = 25; -use arrow::temporal_conversions::{timestamp_ms_to_datetime, timestamp_us_to_datetime}; +#[cfg(any( + feature = "dtype-date", + feature = "dtype-datetime", + feature = "dtype-time" +))] +use arrow::temporal_conversions::*; #[cfg(feature = "fmt")] use comfy_table::presets::{ASCII_FULL, UTF8_FULL}; #[cfg(feature = "fmt")] diff --git a/polars/polars-core/src/frame/chunks.rs b/polars/polars-core/src/frame/chunks.rs index 19f473d7c257..3711a7189496 100644 --- a/polars/polars-core/src/frame/chunks.rs +++ b/polars/polars-core/src/frame/chunks.rs @@ -1,5 +1,4 @@ use crate::prelude::*; -use arrow::array::ArrayRef; use arrow::chunk::Chunk; pub type ArrowChunk = Chunk; diff --git a/polars/polars-core/src/frame/explode.rs b/polars/polars-core/src/frame/explode.rs index e7b00e55fe04..058239c80496 100644 --- a/polars/polars-core/src/frame/explode.rs +++ b/polars/polars-core/src/frame/explode.rs @@ -289,7 +289,7 @@ impl DataFrame { let values = unsafe { Series::from_chunks_and_dtype_unchecked(value_name, vec![values_arr], &st) }; - let variable_col = variable_col.into_arc(); + let variable_col = variable_col.as_box(); // Safety // The give dtype is correct let variables = unsafe { diff --git a/polars/polars-core/src/frame/groupby/aggregations/agg_list.rs b/polars/polars-core/src/frame/groupby/aggregations/agg_list.rs index 5230b3c4da41..5c942efc69b1 100644 --- a/polars/polars-core/src/frame/groupby/aggregations/agg_list.rs +++ b/polars/polars-core/src/frame/groupby/aggregations/agg_list.rs @@ -73,10 +73,16 @@ where validity, ); let data_type = ListArray::::default_datatype(T::get_dtype().to_arrow()); - let arr = - ListArray::::from_data(data_type, offsets.into(), Arc::new(array), None); + // Safety: + // offsets are monotonically increasing + let arr = ListArray::::new_unchecked( + data_type, + offsets.into(), + Box::new(array), + None, + ); - let mut ca = ListChunked::from_chunks(self.name(), vec![Arc::new(arr)]); + let mut ca = ListChunked::from_chunks(self.name(), vec![Box::new(arr)]); if can_fast_explode { ca.set_fast_explode() } @@ -130,9 +136,13 @@ where validity, ); let data_type = ListArray::::default_datatype(T::get_dtype().to_arrow()); - let arr = - ListArray::::from_data(data_type, offsets.into(), Arc::new(array), None); - let mut ca = ListChunked::from_chunks(self.name(), vec![Arc::new(arr)]); + let arr = ListArray::::new_unchecked( + data_type, + offsets.into(), + Box::new(array), + None, + ); + let mut ca = ListChunked::from_chunks(self.name(), vec![Box::new(arr)]); if can_fast_explode { ca.set_fast_explode() } @@ -212,19 +222,21 @@ fn agg_list_list, &mut i64, &mut Vec>(); - let list_values: ArrayRef = arrow::compute::concatenate::concatenate(&arrays) - .unwrap() - .into(); + let list_values: ArrayRef = arrow::compute::concatenate::concatenate(&arrays).unwrap(); let data_type = ListArray::::default_datatype(list_values.data_type().clone()); - let arr = Arc::new(ListArray::::from_data( - data_type, - offsets.into(), - list_values, - None, - )) as ArrayRef; + // Safety: + // offsets are monotonically increasing + let arr = unsafe { + Box::new(ListArray::::new_unchecked( + data_type, + offsets.into(), + list_values, + None, + )) as ArrayRef + }; let mut listarr = ListChunked::from_chunks(ca.name(), vec![arr]); if can_fast_explode { listarr.set_fast_explode() @@ -345,11 +357,13 @@ impl AggList for ObjectChunked { // meaning that the sentinel is heap allocated and the dereference of the // pointer does not fail pe.set_to_series_fn::(); - let extension_array = Arc::new(pe.take_and_forget()) as ArrayRef; + let extension_array = Box::new(pe.take_and_forget()) as ArrayRef; let extension_dtype = extension_array.data_type(); let data_type = ListArray::::default_datatype(extension_dtype.clone()); - let arr = Arc::new(ListArray::::from_data( + // Safety: + // offsets are monotonically increasing + let arr = Box::new(ListArray::::new_unchecked( data_type, offsets.into(), extension_array, diff --git a/polars/polars-core/src/frame/groupby/aggregations/mod.rs b/polars/polars-core/src/frame/groupby/aggregations/mod.rs index a5b8d4c0efda..295e96953277 100644 --- a/polars/polars-core/src/frame/groupby/aggregations/mod.rs +++ b/polars/polars-core/src/frame/groupby/aggregations/mod.rs @@ -58,7 +58,7 @@ where { if values.is_empty() { let out: Vec = vec![]; - return Arc::new(PrimitiveArray::from_data( + return Box::new(PrimitiveArray::from_data( T::PRIMITIVE.into(), out.into(), None, @@ -99,7 +99,7 @@ where }) .collect_trusted::>(); - Arc::new(PrimitiveArray::from_data( + Box::new(PrimitiveArray::from_data( T::PRIMITIVE.into(), out.into(), Some(validity.into()), @@ -119,7 +119,7 @@ where { if values.is_empty() { let out: Vec = vec![]; - return Arc::new(PrimitiveArray::from_data( + return Box::new(PrimitiveArray::from_data( T::PRIMITIVE.into(), out.into(), None, @@ -142,7 +142,7 @@ where }) .collect::>(); - Arc::new(out) + Box::new(out) } fn slice_from_offsets(ca: &ChunkedArray, first: IdxSize, len: IdxSize) -> ChunkedArray diff --git a/polars/polars-core/src/frame/row.rs b/polars/polars-core/src/frame/row.rs index 39f544192d53..d67dce53ef03 100644 --- a/polars/polars-core/src/frame/row.rs +++ b/polars/polars-core/src/frame/row.rs @@ -531,7 +531,7 @@ where validity, ); let name = format!("column_{}", i); - ChunkedArray::::from_chunks(&name, vec![Arc::new(arr) as ArrayRef]).into_series() + ChunkedArray::::from_chunks(&name, vec![Box::new(arr) as ArrayRef]).into_series() }) .collect() }); diff --git a/polars/polars-core/src/series/comparison.rs b/polars/polars-core/src/series/comparison.rs index 67a0ddd1306b..a723f96db8e1 100644 --- a/polars/polars-core/src/series/comparison.rs +++ b/polars/polars-core/src/series/comparison.rs @@ -4,6 +4,12 @@ use super::Series; use crate::apply_method_physical_numeric; use crate::prelude::*; use crate::series::arithmetic::coerce_lhs_rhs; +#[cfg(any( + feature = "dtype-duration", + feature = "dtype-datetime", + feature = "dtype-date", + feature = "dtype-time" +))] use std::ops::Deref; macro_rules! impl_compare { diff --git a/polars/polars-core/src/series/from.rs b/polars/polars-core/src/series/from.rs index ca43fa9d2044..be3d433e98d2 100644 --- a/polars/polars-core/src/series/from.rs +++ b/polars/polars-core/src/series/from.rs @@ -3,9 +3,12 @@ use crate::chunked_array::cast::cast_chunks; use crate::chunked_array::object::extension::polars_extension::PolarsExtension; use crate::prelude::*; use arrow::compute::cast::utf8_to_large_utf8; -use arrow::temporal_conversions::MILLISECONDS; -#[cfg(feature = "dtype-time")] -use arrow::temporal_conversions::NANOSECONDS; +#[cfg(any( + feature = "dtype-date", + feature = "dtype-datetime", + feature = "dtype-time" +))] +use arrow::temporal_conversions::*; use polars_arrow::compute::cast::cast; #[cfg(feature = "dtype-struct")] use polars_arrow::kernels::concatenate::concatenate_owned_unchecked; @@ -263,14 +266,16 @@ impl Series { let offsets = arr.offsets().clone(); let validity = arr.validity().cloned(); - let values = Arc::new(PrimitiveArray::from_data( + let values = Box::new(PrimitiveArray::from_data( ArrowDataType::UInt8, values, None, )); let dtype = ListArray::::default_datatype(ArrowDataType::UInt8); - Arc::new(ListArray::::from_data( + // Safety: + // offsets are monotonically increasing + Box::new(ListArray::::new_unchecked( dtype, offsets, values, validity, )) as ArrayRef }) @@ -311,13 +316,9 @@ impl Series { let new_values = struct_arr .values() .iter() - .map(|arr| { - Arc::from(match arr.validity() { - None => arr.with_validity(Some(validity.clone())), - Some(arr_validity) => { - arr.with_validity(Some(arr_validity & validity)) - } - }) + .map(|arr| match arr.validity() { + None => arr.with_validity(Some(validity.clone())), + Some(arr_validity) => arr.with_validity(Some(arr_validity & validity)), }) .collect(); @@ -352,18 +353,18 @@ fn convert_inner_types(arr: &ArrayRef) -> ArrayRef { match arr.data_type() { ArrowDataType::Utf8 => { let arr = arr.as_any().downcast_ref::>().unwrap(); - Arc::from(utf8_to_large_utf8(arr)) + Box::from(utf8_to_large_utf8(arr)) } ArrowDataType::List(field) => { let out = cast(&**arr, &ArrowDataType::LargeList(field.clone())).unwrap(); - convert_inner_types(&(Arc::from(out) as ArrayRef)) + convert_inner_types(&out) } ArrowDataType::LargeList(_) => { let arr = arr.as_any().downcast_ref::>().unwrap(); let values = convert_inner_types(arr.values()); let dtype = ListArray::::default_datatype(values.data_type().clone()); unsafe { - Arc::from(ListArray::::new_unchecked( + Box::from(ListArray::::new_unchecked( dtype, arr.offsets().clone(), values, @@ -383,7 +384,7 @@ fn convert_inner_types(arr: &ArrayRef) -> ArrayRef { .iter() .map(|f| ArrowField::new(&f.name, DataType::from(&f.data_type).to_arrow(), true)) .collect(); - Arc::new(StructArray::new( + Box::new(StructArray::new( ArrowDataType::Struct(fields), values, arr.validity().cloned(), @@ -429,15 +430,6 @@ impl TryFrom<(&str, ArrayRef)> for Series { } } -impl TryFrom<(&str, Box)> for Series { - type Error = PolarsError; - - fn try_from(name_arr: (&str, Box)) -> Result { - let (name, arr) = name_arr; - Series::try_from((name, vec![arr.into()])) - } -} - pub trait IntoSeries { fn is_series() -> bool { false @@ -510,7 +502,7 @@ mod test { #[cfg(feature = "dtype-u8")] fn test_binary_to_list() { let iter = std::iter::repeat(b"hello").take(2).map(Some); - let a = Arc::new(iter.collect::>()) as ArrayRef; + let a = Box::new(iter.collect::>()) as ArrayRef; let s = Series::try_from(("", a)).unwrap(); assert_eq!(s.dtype(), &DataType::List(Box::new(DataType::UInt8))); diff --git a/polars/polars-core/src/series/implementations/boolean.rs b/polars/polars-core/src/series/implementations/boolean.rs index 6874d2a6685c..00058c671ad6 100644 --- a/polars/polars-core/src/series/implementations/boolean.rs +++ b/polars/polars-core/src/series/implementations/boolean.rs @@ -16,7 +16,6 @@ use crate::frame::hash_join::ZipOuterJoinColumn; use crate::prelude::*; use crate::series::implementations::SeriesWrap; use ahash::RandomState; -use arrow::array::ArrayRef; use polars_arrow::prelude::QuantileInterpolOptions; use std::borrow::Cow; use std::ops::{BitAnd, BitOr, BitXor}; diff --git a/polars/polars-core/src/series/implementations/categorical.rs b/polars/polars-core/src/series/implementations/categorical.rs index b7f6842ce5b0..79485477992a 100644 --- a/polars/polars-core/src/series/implementations/categorical.rs +++ b/polars/polars-core/src/series/implementations/categorical.rs @@ -16,7 +16,6 @@ use crate::frame::hash_join::{check_categorical_src, ZipOuterJoinColumn}; use crate::prelude::*; use crate::series::implementations::SeriesWrap; use ahash::RandomState; -use arrow::array::ArrayRef; use polars_arrow::prelude::QuantileInterpolOptions; use std::borrow::Cow; diff --git a/polars/polars-core/src/series/implementations/floats.rs b/polars/polars-core/src/series/implementations/floats.rs index 97cd17d4b1b9..b20e3474a64b 100644 --- a/polars/polars-core/src/series/implementations/floats.rs +++ b/polars/polars-core/src/series/implementations/floats.rs @@ -19,7 +19,6 @@ use crate::prelude::*; #[cfg(feature = "checked_arithmetic")] use crate::series::arithmetic::checked::NumOpsDispatchChecked; use ahash::RandomState; -use arrow::array::ArrayRef; use polars_arrow::prelude::QuantileInterpolOptions; use std::borrow::Cow; diff --git a/polars/polars-core/src/series/implementations/list.rs b/polars/polars-core/src/series/implementations/list.rs index ead6f6e408fc..37c850e8bac9 100644 --- a/polars/polars-core/src/series/implementations/list.rs +++ b/polars/polars-core/src/series/implementations/list.rs @@ -9,7 +9,6 @@ use crate::prelude::*; use crate::series::implementations::SeriesWrap; #[cfg(feature = "chunked_ids")] use crate::series::IsSorted; -use arrow::array::ArrayRef; use polars_arrow::prelude::QuantileInterpolOptions; use std::any::Any; use std::borrow::Cow; diff --git a/polars/polars-core/src/series/implementations/mod.rs b/polars/polars-core/src/series/implementations/mod.rs index 1f4453c8dd8e..e624cc93e880 100644 --- a/polars/polars-core/src/series/implementations/mod.rs +++ b/polars/polars-core/src/series/implementations/mod.rs @@ -42,7 +42,6 @@ use crate::prelude::*; #[cfg(feature = "checked_arithmetic")] use crate::series::arithmetic::checked::NumOpsDispatchChecked; use ahash::RandomState; -use arrow::array::ArrayRef; use polars_arrow::prelude::QuantileInterpolOptions; use std::borrow::Cow; use std::ops::Deref; diff --git a/polars/polars-core/src/series/implementations/object.rs b/polars/polars-core/src/series/implementations/object.rs index c9f2cba9f2bb..7a3ffebfad0f 100644 --- a/polars/polars-core/src/series/implementations/object.rs +++ b/polars/polars-core/src/series/implementations/object.rs @@ -7,7 +7,6 @@ use crate::series::implementations::SeriesWrap; use crate::series::private::{PrivateSeries, PrivateSeriesNumeric}; use crate::series::IsSorted; use ahash::RandomState; -use arrow::array::ArrayRef; use std::any::Any; use std::borrow::Cow; diff --git a/polars/polars-core/src/series/implementations/utf8.rs b/polars/polars-core/src/series/implementations/utf8.rs index 2c6f617390e2..3985cb456180 100644 --- a/polars/polars-core/src/series/implementations/utf8.rs +++ b/polars/polars-core/src/series/implementations/utf8.rs @@ -16,7 +16,6 @@ use crate::frame::hash_join::ZipOuterJoinColumn; use crate::prelude::*; use crate::series::implementations::SeriesWrap; use ahash::RandomState; -use arrow::array::ArrayRef; use polars_arrow::prelude::QuantileInterpolOptions; use std::borrow::Cow; diff --git a/polars/polars-core/src/series/into.rs b/polars/polars-core/src/series/into.rs index 838229ad0aca..03ab19aed73e 100644 --- a/polars/polars-core/src/series/into.rs +++ b/polars/polars-core/src/series/into.rs @@ -34,28 +34,20 @@ impl Series { let new = CategoricalChunked::from_cats_and_rev_map(cats, ca.get_rev_map().clone()); let arr: DictionaryArray = (&new).into(); - Arc::new(arr) as ArrayRef + Box::new(arr) as ArrayRef } #[cfg(feature = "dtype-date")] - DataType::Date => { - let arr = cast(&*self.chunks()[chunk_idx], &DataType::Date.to_arrow()).unwrap(); - Arc::from(arr) - } + DataType::Date => cast(&*self.chunks()[chunk_idx], &DataType::Date.to_arrow()).unwrap(), #[cfg(feature = "dtype-datetime")] DataType::Datetime(_, _) => { - let arr = cast(&*self.chunks()[chunk_idx], &self.dtype().to_arrow()).unwrap(); - Arc::from(arr) + cast(&*self.chunks()[chunk_idx], &self.dtype().to_arrow()).unwrap() } #[cfg(feature = "dtype-duration")] DataType::Duration(_) => { - let arr = cast(&*self.chunks()[chunk_idx], &self.dtype().to_arrow()).unwrap(); - Arc::from(arr) + cast(&*self.chunks()[chunk_idx], &self.dtype().to_arrow()).unwrap() } #[cfg(feature = "dtype-time")] - DataType::Time => { - let arr = cast(&*self.chunks()[chunk_idx], &DataType::Time.to_arrow()).unwrap(); - Arc::from(arr) - } + DataType::Time => cast(&*self.chunks()[chunk_idx], &DataType::Time.to_arrow()).unwrap(), #[cfg(feature = "dtype-struct")] DataType::Struct(_) => self.array_ref(chunk_idx).clone(), _ => self.array_ref(chunk_idx).clone(), diff --git a/polars/polars-core/src/series/mod.rs b/polars/polars-core/src/series/mod.rs index 3e2bde563638..2f2362b06e99 100644 --- a/polars/polars-core/src/series/mod.rs +++ b/polars/polars-core/src/series/mod.rs @@ -1,7 +1,6 @@ //! Type agnostic columnar data structure. pub use crate::prelude::ChunkCompare; use crate::prelude::*; -use arrow::array::ArrayRef; #[cfg(any(feature = "dtype-struct", feature = "object"))] use std::any::Any; @@ -19,9 +18,11 @@ pub mod unstable; #[cfg(feature = "rank")] use crate::prelude::unique::rank::rank; +#[cfg(feature = "zip_with")] +use crate::series::arithmetic::coerce_lhs_rhs; use crate::utils::{split_ca, split_series}; use crate::utils::{split_offsets, Wrap}; -use crate::{series::arithmetic::coerce_lhs_rhs, POOL}; +use crate::POOL; use ahash::RandomState; use arrow::compute::aggregate::estimated_bytes_size; pub use from::*; @@ -709,10 +710,9 @@ impl Series { { panic!("activate feature dtype-date") } + #[cfg(feature = "dtype-date")] match self.dtype() { - #[cfg(feature = "dtype-date")] DataType::Int32 => self.i32().unwrap().clone().into_date().into_series(), - #[cfg(feature = "dtype-date")] DataType::Date => self .date() .unwrap() @@ -729,15 +729,14 @@ impl Series { panic!("activate feature dtype-datetime") } + #[cfg(feature = "dtype-datetime")] match self.dtype() { - #[cfg(feature = "dtype-datetime")] DataType::Int64 => self .i64() .unwrap() .clone() .into_datetime(timeunit, tz) .into_series(), - #[cfg(feature = "dtype-datetime")] DataType::Datetime(_, _) => self .datetime() .unwrap() @@ -754,15 +753,14 @@ impl Series { { panic!("activate feature dtype-duration") } + #[cfg(feature = "dtype-duration")] match self.dtype() { - #[cfg(feature = "dtype-duration")] DataType::Int64 => self .i64() .unwrap() .clone() .into_duration(timeunit) .into_series(), - #[cfg(feature = "dtype-duration")] DataType::Duration(_) => self .duration() .unwrap() @@ -1018,7 +1016,7 @@ mod test { #[test] fn new_series_from_arrow_primitive_array() { let array = UInt32Array::from_slice(&[1, 2, 3, 4, 5]); - let array_ref: ArrayRef = Arc::new(array); + let array_ref: ArrayRef = Box::new(array); let _ = Series::try_from(("foo", array_ref)).unwrap(); } diff --git a/polars/polars-core/src/series/ops/ewm.rs b/polars/polars-core/src/series/ops/ewm.rs index f9aa089fda0f..b8799f57a48c 100644 --- a/polars/polars-core/src/series/ops/ewm.rs +++ b/polars/polars-core/src/series/ops/ewm.rs @@ -41,7 +41,7 @@ impl Series { ewma_inf_hist_no_nulls(vals.iter().copied(), options.alpha as f32) }; let arr = prepare_primitive_array(out, options.min_periods, 0); - Series::try_from((self.name(), Arc::new(arr) as ArrayRef)) + Series::try_from((self.name(), Box::new(arr) as ArrayRef)) } _ => { let iter = ca.into_no_null_iter(); @@ -51,7 +51,7 @@ impl Series { ewma_inf_hist_no_nulls(iter, options.alpha as f32) }; let arr = prepare_primitive_array(out, options.min_periods, 0); - Series::try_from((self.name(), Arc::new(arr) as ArrayRef)) + Series::try_from((self.name(), Box::new(arr) as ArrayRef)) } } } @@ -67,7 +67,7 @@ impl Series { ewma_inf_hist_no_nulls(vals.iter().copied(), options.alpha) }; let arr = prepare_primitive_array(out, options.min_periods, 0); - Series::try_from((self.name(), Arc::new(arr) as ArrayRef)) + Series::try_from((self.name(), Box::new(arr) as ArrayRef)) } _ => { let iter = ca.into_no_null_iter(); @@ -77,7 +77,7 @@ impl Series { ewma_inf_hist_no_nulls(iter, options.alpha) }; let arr = prepare_primitive_array(out, options.min_periods, 0); - Series::try_from((self.name(), Arc::new(arr) as ArrayRef)) + Series::try_from((self.name(), Box::new(arr) as ArrayRef)) } } } @@ -90,7 +90,7 @@ impl Series { ewma_inf_hists(iter, options.alpha as f32) }; let arr = prepare_primitive_array(out, options.min_periods, leading_nulls); - Series::try_from((self.name(), Arc::new(arr) as ArrayRef)) + Series::try_from((self.name(), Box::new(arr) as ArrayRef)) } (DataType::Float64, _) => { let ca = self.f64().unwrap(); @@ -101,7 +101,7 @@ impl Series { ewma_inf_hists(iter, options.alpha) }; let arr = prepare_primitive_array(out, options.min_periods, leading_nulls); - Series::try_from((self.name(), Arc::new(arr) as ArrayRef)) + Series::try_from((self.name(), Box::new(arr) as ArrayRef)) } _ => self.cast(&DataType::Float64)?.ewm_mean(options), } @@ -126,7 +126,7 @@ impl Series { ewm_std(x_slice, ewma_slice, options.alpha); // we mask the original null values until we know better how to deal with them. - let out = Arc::new(ewma_arr.with_validity(arr.validity().cloned())) as ArrayRef; + let out = Box::new(ewma_arr.with_validity(arr.validity().cloned())) as ArrayRef; Series::try_from((self.name(), out)) } DataType::Float32 => { @@ -139,7 +139,7 @@ impl Series { ewm_std(x_slice, ewma_slice, options.alpha as f32); // we mask the original null values until we know better how to deal with them. - let out = Arc::new(ewma_arr.with_validity(arr.validity().cloned())) as ArrayRef; + let out = Box::new(ewma_arr.with_validity(arr.validity().cloned())) as ArrayRef; Series::try_from((self.name(), out)) } _ => unimplemented!(), @@ -165,7 +165,7 @@ impl Series { ewm_var(x_slice, ewma_slice, options.alpha); // we mask the original null values until we know better how to deal with them. - let out = Arc::new(ewma_arr.with_validity(arr.validity().cloned())) as ArrayRef; + let out = Box::new(ewma_arr.with_validity(arr.validity().cloned())) as ArrayRef; Series::try_from((self.name(), out)) } DataType::Float32 => { @@ -178,7 +178,7 @@ impl Series { ewm_var(x_slice, ewma_slice, options.alpha as f32); // we mask the original null values until we know better how to deal with them. - let out = Arc::new(ewma_arr.with_validity(arr.validity().cloned())) as ArrayRef; + let out = Box::new(ewma_arr.with_validity(arr.validity().cloned())) as ArrayRef; Series::try_from((self.name(), out)) } _ => unimplemented!(), diff --git a/polars/polars-core/src/series/ops/to_list.rs b/polars/polars-core/src/series/ops/to_list.rs index 09f4eb954d13..ecccbc0371b2 100644 --- a/polars/polars-core/src/series/ops/to_list.rs +++ b/polars/polars-core/src/series/ops/to_list.rs @@ -7,12 +7,12 @@ fn reshape_fast_path(name: &str, s: &Series) -> Series { let chunks = match s.dtype() { #[cfg(feature = "dtype-struct")] DataType::Struct(_) => { - vec![Arc::new(array_to_unit_list(s.array_ref(0).clone())) as ArrayRef] + vec![Box::new(array_to_unit_list(s.array_ref(0).clone())) as ArrayRef] } _ => s .chunks() .iter() - .map(|arr| Arc::new(array_to_unit_list(arr.clone())) as ArrayRef) + .map(|arr| Box::new(array_to_unit_list(arr.clone())) as ArrayRef) .collect::>(), }; @@ -34,10 +34,13 @@ impl Series { let data_type = ListArray::::default_datatype(inner_type.to_physical().to_arrow()); - let arr = ListArray::from_data(data_type, offsets.into(), values.clone(), None); + // Safety: + // offsets are correct; + let arr = + unsafe { ListArray::new_unchecked(data_type, offsets.into(), values.clone(), None) }; let name = self.name(); - let mut ca = ListChunked::from_chunks(name, vec![Arc::new(arr)]); + let mut ca = ListChunked::from_chunks(name, vec![Box::new(arr)]); if self.dtype() != &self.dtype().to_physical() { ca.to_logical(inner_type.clone()) } diff --git a/polars/polars-core/src/series/series_trait.rs b/polars/polars-core/src/series/series_trait.rs index 9cfe7a7bdc74..a631cf3ab0ff 100644 --- a/polars/polars-core/src/series/series_trait.rs +++ b/polars/polars-core/src/series/series_trait.rs @@ -2,7 +2,6 @@ use crate::chunked_array::object::PolarsObjectSafe; pub use crate::prelude::ChunkCompare; use crate::prelude::*; -use arrow::array::ArrayRef; use polars_arrow::prelude::QuantileInterpolOptions; use std::any::Any; use std::borrow::Cow; diff --git a/polars/polars-core/src/vector_hasher.rs b/polars/polars-core/src/vector_hasher.rs index f06757561262..2aeb98f7398e 100644 --- a/polars/polars-core/src/vector_hasher.rs +++ b/polars/polars-core/src/vector_hasher.rs @@ -529,7 +529,7 @@ pub(crate) fn df_rows_to_hashes( keys.vec_hash_combine(build_hasher.clone(), hslice); } - let chunks = vec![Arc::new(PrimitiveArray::from_data( + let chunks = vec![Box::new(PrimitiveArray::from_data( ArrowDataType::UInt64, hashes.into(), None, diff --git a/polars/polars-io/Cargo.toml b/polars/polars-io/Cargo.toml index c174ec65a799..79389f12c3b8 100644 --- a/polars/polars-io/Cargo.toml +++ b/polars/polars-io/Cargo.toml @@ -35,8 +35,8 @@ private = ["polars-time/private"] [dependencies] ahash = "0.7" anyhow = "1.0" -arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "6919cc3281c3ee7dd3e973a7b303c67277458f20", default-features = false } -# arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "improve_mutable", default-features = false } +arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "39db6fb7514364bfea08d594793b23e1ed5a7def", default-features = false } +# arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "count_shared", default-features = false } # arrow = { package = "arrow2", version = "0.12", default-features = false } # arrow = { package = "arrow2", path = "../../../arrow2", default-features = false } csv-core = { version = "0.1.10", optional = true } diff --git a/polars/polars-io/src/csv_core/buffer.rs b/polars/polars-io/src/csv_core/buffer.rs index 7185b2b4de30..5a12177d499b 100644 --- a/polars/polars-io/src/csv_core/buffer.rs +++ b/polars/polars-io/src/csv_core/buffer.rs @@ -493,7 +493,7 @@ impl Buffer { v.data.into(), Some(v.validity.into()), ); - let ca = Utf8Chunked::from_chunks(&v.name, vec![Arc::new(arr)]); + let ca = Utf8Chunked::from_chunks(&v.name, vec![Box::new(arr)]); ca.into_series() }, }; diff --git a/polars/polars-io/src/json.rs b/polars/polars-io/src/json.rs index cde71d3b86bc..bd6a87fe4c5e 100644 --- a/polars/polars-io/src/json.rs +++ b/polars/polars-io/src/json.rs @@ -63,7 +63,7 @@ //! use crate::mmap::{MmapBytesReader, ReaderBytes}; use crate::prelude::*; -use arrow::array::{ArrayRef, StructArray}; +use arrow::array::StructArray; use arrow::io::ndjson::read::FallibleStreamingIterator; pub use arrow::{ error::Result as ArrowResult, @@ -112,7 +112,7 @@ where let fields = df.iter().map(|s| s.field().to_arrow()).collect::>(); let batches = df .iter_chunks() - .map(|chunk| Ok(Arc::new(chunk_to_struct(chunk, fields.clone())) as ArrayRef)); + .map(|chunk| Ok(Box::new(chunk_to_struct(chunk, fields.clone())) as ArrayRef)); match self.json_format { JsonFormat::JsonLines => { diff --git a/polars/polars-io/src/lib.rs b/polars/polars-io/src/lib.rs index 9a78e07008b1..9b3ef0f7eac6 100644 --- a/polars/polars-io/src/lib.rs +++ b/polars/polars-io/src/lib.rs @@ -150,10 +150,7 @@ pub(crate) fn finish_reader( .fields .iter() .map(|fld| { - Series::try_from(( - fld.name.as_str(), - Arc::from(new_empty_array(fld.data_type.clone())), - )) + Series::try_from((fld.name.as_str(), new_empty_array(fld.data_type.clone()))) }) .collect::>()?; DataFrame::new(empty_cols)? diff --git a/polars/polars-io/src/parquet/predicates.rs b/polars/polars-io/src/parquet/predicates.rs index 9bf5dcdf139d..c5eb947a05f6 100644 --- a/polars/polars-io/src/parquet/predicates.rs +++ b/polars/polars-io/src/parquet/predicates.rs @@ -1,5 +1,5 @@ use crate::ArrowResult; -use arrow::array::{Array, ArrayRef}; +use arrow::array::Array; use arrow::compute::concatenate::concatenate; use arrow::io::parquet::read::statistics::{self, deserialize, Statistics}; use arrow::io::parquet::read::RowGroupMetaData; @@ -38,7 +38,7 @@ impl ColumnStats { let dtype = DataType::from(min_val.data_type()); if dtype.is_numeric() || matches!(dtype, DataType::Utf8) { let arr = concatenate(&[min_val, max_val]).unwrap(); - let s = Series::try_from(("", Arc::from(arr) as ArrayRef)).unwrap(); + let s = Series::try_from(("", arr)).unwrap(); if s.null_count() > 0 { None } else { diff --git a/polars/polars-io/src/parquet/read_impl.rs b/polars/polars-io/src/parquet/read_impl.rs index cd65779f9ca6..7d5ffad23d05 100644 --- a/polars/polars-io/src/parquet/read_impl.rs +++ b/polars/polars-io/src/parquet/read_impl.rs @@ -20,7 +20,7 @@ use std::sync::Arc; fn array_iter_to_series(iter: ArrayIter, field: &ArrowField) -> Result { let chunks = iter.collect::>>()?; if chunks.is_empty() { - let arr = Arc::from(new_empty_array(field.data_type.clone())); + let arr = new_empty_array(field.data_type.clone()); Series::try_from((field.name.as_str(), arr)) } else { Series::try_from((field.name.as_str(), chunks)) diff --git a/polars/polars-io/src/parquet/write.rs b/polars/polars-io/src/parquet/write.rs index 416463ad6d70..5f169613ab6a 100644 --- a/polars/polars-io/src/parquet/write.rs +++ b/polars/polars-io/src/parquet/write.rs @@ -133,7 +133,7 @@ where } fn create_serializer( - batch: Chunk>, + batch: Chunk>, fields: Vec, encodings: Vec>, options: WriteOptions, diff --git a/polars/polars-lazy/src/dsl/string.rs b/polars/polars-lazy/src/dsl/string.rs index 02796e9367f6..2481484d502e 100644 --- a/polars/polars-lazy/src/dsl/string.rs +++ b/polars/polars-lazy/src/dsl/string.rs @@ -229,8 +229,8 @@ impl StringNameSpace { let fields = arrs .into_iter() .enumerate() - .map(|(i, arr)| { - Series::try_from((format!("field_{i}").as_str(), arr.into_arc())).unwrap() + .map(|(i, mut arr)| { + Series::try_from((format!("field_{i}").as_str(), arr.as_box())).unwrap() }) .collect::>(); Ok(StructChunked::new(ca.name(), &fields)?.into_series()) @@ -281,8 +281,8 @@ impl StringNameSpace { let fields = arrs .into_iter() .enumerate() - .map(|(i, arr)| { - Series::try_from((format!("field_{i}").as_str(), arr.into_arc())).unwrap() + .map(|(i, mut arr)| { + Series::try_from((format!("field_{i}").as_str(), arr.as_box())).unwrap() }) .collect::>(); Ok(StructChunked::new(ca.name(), &fields)?.into_series()) diff --git a/polars/polars-lazy/src/physical_plan/expressions/aggregation.rs b/polars/polars-lazy/src/physical_plan/expressions/aggregation.rs index 6b16731f7d9b..2ff11427a5ab 100644 --- a/polars/polars-lazy/src/physical_plan/expressions/aggregation.rs +++ b/polars/polars-lazy/src/physical_plan/expressions/aggregation.rs @@ -333,15 +333,19 @@ impl PartitionedAggregation for AggregationExpr { } let vals = values.iter().map(|arr| &**arr).collect::>(); - let values: ArrayRef = concatenate(&vals).unwrap().into(); + let values = concatenate(&vals).unwrap(); let data_type = ListArray::::default_datatype(values.data_type().clone()); - let arr = Arc::new(ListArray::::from_data( - data_type, - offsets.into(), - values, - None, - )) as ArrayRef; + // Safety: + // offsets are monotonically increasing + let arr = unsafe { + Box::new(ListArray::::new_unchecked( + data_type, + offsets.into(), + values, + None, + )) as ArrayRef + }; let mut ca = ListChunked::from_chunks(&new_name, vec![arr]); if can_fast_explode { ca.set_fast_explode() diff --git a/polars/polars-lazy/src/physical_plan/expressions/binary.rs b/polars/polars-lazy/src/physical_plan/expressions/binary.rs index 2427793e5e82..bf0dacef9367 100644 --- a/polars/polars-lazy/src/physical_plan/expressions/binary.rs +++ b/polars/polars-lazy/src/physical_plan/expressions/binary.rs @@ -174,7 +174,7 @@ impl PhysicalExpr for BinaryExpr { // Safety: // we are in bounds - let arr = unsafe { Arc::from(arr_l.slice_unchecked(idx, 1)) }; + let arr = unsafe { arr_l.slice_unchecked(idx, 1) }; us.swap(arr); let l = us.as_ref(); @@ -224,7 +224,7 @@ impl PhysicalExpr for BinaryExpr { // TODO: optimize this? Its slow. // Safety: // we are in bounds - let arr = unsafe { Arc::from(arr_r.slice_unchecked(idx, 1)) }; + let arr = unsafe { arr_r.slice_unchecked(idx, 1) }; us.swap(arr); let r = us.as_ref(); diff --git a/polars/polars-lazy/src/physical_plan/expressions/group_iter.rs b/polars/polars-lazy/src/physical_plan/expressions/group_iter.rs index d0423d29d855..48e48a294286 100644 --- a/polars/polars-lazy/src/physical_plan/expressions/group_iter.rs +++ b/polars/polars-lazy/src/physical_plan/expressions/group_iter.rs @@ -1,5 +1,4 @@ use super::*; -use polars_arrow::export::arrow::array::ArrayRef; use polars_core::series::unstable::UnstableSeries; use std::pin::Pin; @@ -94,7 +93,7 @@ impl<'a> Iterator for FlatIter<'a> { if self.len == self.offset { None } else { - let arr = unsafe { Arc::from(self.array.slice_unchecked(self.offset, 1)) }; + let arr = unsafe { self.array.slice_unchecked(self.offset, 1) }; self.offset += 1; self.item.swap(arr); Some(Some(self.item)) diff --git a/polars/polars-ops/src/chunked_array/strings/namespace.rs b/polars/polars-ops/src/chunked_array/strings/namespace.rs index 8e9c4e83641f..62326d92a86c 100644 --- a/polars/polars-ops/src/chunked_array/strings/namespace.rs +++ b/polars/polars-ops/src/chunked_array/strings/namespace.rs @@ -202,7 +202,7 @@ pub trait Utf8NameSpaceImpl: AsUtf8 { let ca = self.as_utf8(); let chunks = ca .downcast_iter() - .map(|c| Ok(substring(c, start, &length)?.into())) + .map(|c| substring(c, start, &length)) .collect::>()?; Ok(Utf8Chunked::from_chunks(ca.name(), chunks)) diff --git a/polars/polars-time/src/chunkedarray/datetime.rs b/polars/polars-time/src/chunkedarray/datetime.rs index e3bc7db0486e..a4b7c12de64a 100644 --- a/polars/polars-time/src/chunkedarray/datetime.rs +++ b/polars/polars-time/src/chunkedarray/datetime.rs @@ -1,5 +1,5 @@ use super::*; -use arrow::array::{Array, ArrayRef, PrimitiveArray}; +use arrow::array::{Array, PrimitiveArray}; use arrow::compute::cast::CastOptions; use arrow::compute::{cast::cast, temporal}; use arrow::error::Result as ArrowResult; @@ -26,7 +26,7 @@ fn cast_and_apply< }, ) .unwrap(); - Arc::from(func(&*arr).unwrap()) as ArrayRef + Box::from(func(&*arr).unwrap()) as ArrayRef }) .collect(); diff --git a/polars/polars-time/src/chunkedarray/kernels.rs b/polars/polars-time/src/chunkedarray/kernels.rs index 30f22208039a..e00cc46f295a 100644 --- a/polars/polars-time/src/chunkedarray/kernels.rs +++ b/polars/polars-time/src/chunkedarray/kernels.rs @@ -2,7 +2,7 @@ //! `week`, `weekday`, `year`, `hour` etc. from primitive arrays. use super::*; use chrono::{Datelike, NaiveDate, NaiveDateTime, Timelike}; -use polars_arrow::export::arrow::array::{ArrayRef, PrimitiveArray}; +use polars_arrow::export::arrow::array::PrimitiveArray; use polars_arrow::export::arrow::compute::arity::unary; #[cfg(feature = "dtype-time")] use polars_arrow::export::arrow::temporal_conversions::time64ns_to_time; @@ -10,7 +10,6 @@ use polars_arrow::export::arrow::temporal_conversions::{ date32_to_datetime, timestamp_ms_to_datetime, timestamp_ns_to_datetime, timestamp_us_to_datetime, }; -use std::sync::Arc; trait PolarsWeekDay { fn p_weekday(&self) -> u32; @@ -38,7 +37,7 @@ impl PolarsWeekDay for NaiveDate { macro_rules! to_temporal_unit { ($name: ident, $chrono_method:ident, $to_datetime_fn: expr, $dtype_in: ty, $dtype_out:expr) => { pub(crate) fn $name(arr: &PrimitiveArray<$dtype_in>) -> ArrayRef { - Arc::new(unary( + Box::new(unary( arr, |value| { let dt = $to_datetime_fn(value); diff --git a/polars/polars-time/src/chunkedarray/rolling_window/mod.rs b/polars/polars-time/src/chunkedarray/rolling_window/mod.rs index 30f17d2bd7a8..5876df8d47dc 100644 --- a/polars/polars-time/src/chunkedarray/rolling_window/mod.rs +++ b/polars/polars-time/src/chunkedarray/rolling_window/mod.rs @@ -6,7 +6,7 @@ mod rolling_kernels; use crate::prelude::*; use crate::series::WrapFloat; #[cfg(feature = "rolling_window")] -use arrow::array::{Array, ArrayRef, PrimitiveArray}; +use arrow::array::{Array, PrimitiveArray}; use polars_arrow::data_types::IsFloat; #[cfg(feature = "rolling_window")] use polars_arrow::export::arrow; diff --git a/polars/polars-time/src/chunkedarray/rolling_window/rolling_kernels/mod.rs b/polars/polars-time/src/chunkedarray/rolling_window/rolling_kernels/mod.rs index a72c3c4b9429..14bb21a49560 100644 --- a/polars/polars-time/src/chunkedarray/rolling_window/rolling_kernels/mod.rs +++ b/polars/polars-time/src/chunkedarray/rolling_window/rolling_kernels/mod.rs @@ -1,6 +1,5 @@ pub(super) mod no_nulls; use crate::prelude::*; -use arrow::array::ArrayRef; use arrow::array::PrimitiveArray; use arrow::types::NativeType; use polars_arrow::data_types::IsFloat; @@ -11,4 +10,3 @@ use polars_core::export::num::{Bounded, Float, NumCast, One}; use polars_core::prelude::*; use std::fmt::Debug; use std::ops::{AddAssign, Div, Mul, Sub, SubAssign}; -use std::sync::Arc; diff --git a/polars/polars-time/src/chunkedarray/rolling_window/rolling_kernels/no_nulls.rs b/polars/polars-time/src/chunkedarray/rolling_window/rolling_kernels/no_nulls.rs index 5c0f3a7a44ea..cc7c66299c59 100644 --- a/polars/polars-time/src/chunkedarray/rolling_window/rolling_kernels/no_nulls.rs +++ b/polars/polars-time/src/chunkedarray/rolling_window/rolling_kernels/no_nulls.rs @@ -12,7 +12,7 @@ where { if values.is_empty() { let out: Vec = vec![]; - return Arc::new(PrimitiveArray::from_data( + return Box::new(PrimitiveArray::from_data( T::PRIMITIVE.into(), out.into(), None, @@ -35,7 +35,7 @@ where }) .collect::>(); - Arc::new(out) + Box::new(out) } pub(crate) fn rolling_min( diff --git a/polars/polars-time/src/chunkedarray/time.rs b/polars/polars-time/src/chunkedarray/time.rs index b5ea7b793482..7a7920e05a08 100644 --- a/polars/polars-time/src/chunkedarray/time.rs +++ b/polars/polars-time/src/chunkedarray/time.rs @@ -63,7 +63,7 @@ impl TimeMethods for TimeChunked { } let arr: Utf8Array = mutarr.into(); - Arc::new(arr) + Box::new(arr) }); ca.rename(self.name()); diff --git a/polars/polars-time/src/chunkedarray/utf8/infer.rs b/polars/polars-time/src/chunkedarray/utf8/infer.rs index 471f6da87891..7fc5c513a15c 100644 --- a/polars/polars-time/src/chunkedarray/utf8/infer.rs +++ b/polars/polars-time/src/chunkedarray/utf8/infer.rs @@ -4,7 +4,7 @@ use crate::chunkedarray::date::naive_date_to_date; use crate::chunkedarray::utf8::patterns::Pattern; use crate::chunkedarray::utf8::strptime; use chrono::{NaiveDate, NaiveDateTime}; -use polars_arrow::export::arrow::array::{ArrayRef, PrimitiveArray}; +use polars_arrow::export::arrow::array::PrimitiveArray; use polars_core::prelude::*; use polars_core::utils::arrow::types::NativeType; @@ -124,7 +124,7 @@ impl DatetimeInfer { let iter = array .into_iter() .map(|opt_val| opt_val.and_then(|val| self.parse(val))); - Arc::new(PrimitiveArray::from_trusted_len_iter(iter)) as ArrayRef + Box::new(PrimitiveArray::from_trusted_len_iter(iter)) as ArrayRef }) .collect(); let mut out = match self.logical_type { diff --git a/py-polars/Cargo.lock b/py-polars/Cargo.lock index 34b92bc2c0ba..28995536f61d 100644 --- a/py-polars/Cargo.lock +++ b/py-polars/Cargo.lock @@ -74,7 +74,7 @@ dependencies = [ [[package]] name = "arrow2" version = "0.12.0" -source = "git+https://github.com/jorgecarleitao/arrow2?rev=6919cc3281c3ee7dd3e973a7b303c67277458f20#6919cc3281c3ee7dd3e973a7b303c67277458f20" +source = "git+https://github.com/jorgecarleitao/arrow2?rev=39db6fb7514364bfea08d594793b23e1ed5a7def#39db6fb7514364bfea08d594793b23e1ed5a7def" dependencies = [ "arrow-format", "avro-schema", @@ -82,6 +82,7 @@ dependencies = [ "bytemuck", "chrono", "crc", + "dyn-clone", "either", "fallible-streaming-iterator", "futures", @@ -125,9 +126,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.53" +version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" +checksum = "96cf8829f67d2eab0b2dfa42c5d0ef737e0724e4a82b01b3e292456202b19716" dependencies = [ "proc-macro2", "quote", @@ -409,6 +410,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "dyn-clone" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21e50f3adc76d6a43f5ed73b698a87d0760ca74617f60f7c3b879003536fdd28" + [[package]] name = "either" version = "1.6.1" @@ -423,13 +430,11 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "flate2" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" dependencies = [ - "cfg-if", "crc32fast", - "libc", "libz-sys", "miniz_oxide", ] @@ -531,13 +536,13 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" +checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" dependencies = [ "cfg-if", "libc", - "wasi 0.10.2+wasi-snapshot-preview1", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] @@ -605,9 +610,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "indexmap" -version = "1.8.1" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" +checksum = "e6012d540c5baa3589337a98ce73408de9b5a25ec9fc2c6fd6be8f0d39e0ca5a" dependencies = [ "autocfg", "hashbrown 0.11.2", @@ -700,18 +705,18 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "lexical" -version = "6.1.0" +version = "6.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccd3e434c16f0164124ade12dcdee324fcc3dafb1cad0c7f1d8c2451a1aa6886" +checksum = "c7aefb36fd43fef7003334742cbf77b243fcd36418a1d1bdd480d613a67968f6" dependencies = [ "lexical-core", ] [[package]] name = "lexical-core" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92912c4af2e7d9075be3e5e3122c4d7263855fa6cce34fbece4dd08e5884624d" +checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" dependencies = [ "lexical-parse-float", "lexical-parse-integer", @@ -722,9 +727,9 @@ dependencies = [ [[package]] name = "lexical-parse-float" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f518eed87c3be6debe6d26b855c97358d8a11bf05acec137e5f53080f5ad2dd8" +checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" dependencies = [ "lexical-parse-integer", "lexical-util", @@ -733,9 +738,9 @@ dependencies = [ [[package]] name = "lexical-parse-integer" -version = "0.8.3" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc852ec67c6538bbb2b9911116a385b24510e879a69ab516e6a151b15a79168" +checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" dependencies = [ "lexical-util", "static_assertions", @@ -743,18 +748,18 @@ dependencies = [ [[package]] name = "lexical-util" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c72a9d52c5c4e62fa2cdc2cb6c694a39ae1382d9c2a17a466f18e272a0930eb1" +checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" dependencies = [ "static_assertions", ] [[package]] name = "lexical-write-float" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a89ec1d062e481210c309b672f73a0567b7855f21e7d2fae636df44d12e97f9" +checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" dependencies = [ "lexical-util", "lexical-write-integer", @@ -763,9 +768,9 @@ dependencies = [ [[package]] name = "lexical-write-integer" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "094060bd2a7c2ff3a16d5304a6ae82727cb3cc9d1c70f813cc73f744c319337e" +checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" dependencies = [ "lexical-util", "static_assertions", @@ -814,9 +819,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.6" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e7e15d7610cce1d9752e137625f14e61a28cd45929b6e12e47b50fe154ee2e" +checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" dependencies = [ "cc", "cmake", @@ -881,9 +886,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memmap2" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057a3db23999c867821a7a59feb06a578fcb03685e983dff90daf9e7d24ac08f" +checksum = "d5172b50c23043ff43dd53e51392f36519d9b35a8f3a410d30ece5d1aedd58ae" dependencies = [ "libc", ] @@ -908,9 +913,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.5.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082" +checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" dependencies = [ "adler", ] @@ -1062,15 +1067,15 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.10.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" +checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" [[package]] name = "parking_lot" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", "parking_lot_core", @@ -1091,9 +1096,9 @@ dependencies = [ [[package]] name = "parquet-format-async-temp" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488c8b5f43521d019fade4bcc0ce88cce5da5fd26eb1d38b933807041f5930bf" +checksum = "e1a672c84c3e5b5eb6530286b2d22cc1ea8e1e3560e4c314218d6ab749c6db99" dependencies = [ "async-trait", "futures", @@ -1102,9 +1107,9 @@ dependencies = [ [[package]] name = "parquet2" -version = "0.13.0" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5653509c49578c1dbea76a1007d6e3b2fbcf52f2fcb9a3d84593599510c293ae" +checksum = "73fd2690ad041f9296876daef1f2706f6347073bdbcc719090887f1691e4a09d" dependencies = [ "async-stream", "bitpacking", @@ -1465,9 +1470,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.5" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" +checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1" dependencies = [ "aho-corasick", "memchr", @@ -1476,9 +1481,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.25" +version = "0.6.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" [[package]] name = "rle-decode-fast" @@ -1638,9 +1643,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.95" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbaf6116ab8924f39d52792136fb74fd60a80194cf1b1c6ffa6453eef1c3f942" +checksum = "0748dd251e24453cb8717f0354206b91557e4ec8703673a4b30208f2abaf1ebf" dependencies = [ "proc-macro2", "quote", @@ -1649,9 +1654,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7fa7e55043acb85fca6b3c01485a2eeb6b69c5d21002e273c79e465f43b7ac1" +checksum = "c02424087780c9b71cc96799eaeddff35af2bc513278cda5c99fc1f5d026d3c1" [[package]] name = "thiserror" @@ -1675,19 +1680,20 @@ dependencies = [ [[package]] name = "time" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", + "wasi 0.10.0+wasi-snapshot-preview1", "winapi", ] [[package]] name = "unicode-ident" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee" +checksum = "5bd2fe26506023ed7b5e1e315add59d6f584c621d037f9368fea9cfb988f368c" [[package]] name = "unicode-segmentation" @@ -1721,9 +1727,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "wasi" -version = "0.10.2+wasi-snapshot-preview1" +version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasi" diff --git a/py-polars/src/arrow_interop/to_py.rs b/py-polars/src/arrow_interop/to_py.rs index 08ab7f4859bc..a245666484e6 100644 --- a/py-polars/src/arrow_interop/to_py.rs +++ b/py-polars/src/arrow_interop/to_py.rs @@ -1,6 +1,6 @@ -use polars::prelude::ArrowField; +use polars::prelude::{ArrayRef, ArrowField}; use polars_core::frame::ArrowChunk; -use polars_core::utils::arrow::{array::ArrayRef, ffi}; +use polars_core::utils::arrow::ffi; use pyo3::ffi::Py_uintptr_t; use pyo3::prelude::*; diff --git a/py-polars/src/arrow_interop/to_rust.rs b/py-polars/src/arrow_interop/to_rust.rs index 6b75616766fe..acadce6050bc 100644 --- a/py-polars/src/arrow_interop/to_rust.rs +++ b/py-polars/src/arrow_interop/to_rust.rs @@ -2,7 +2,7 @@ use crate::error::PyPolarsErr; use polars_core::export::rayon::prelude::*; use polars_core::prelude::*; use polars_core::utils::accumulate_dataframes_vertical; -use polars_core::utils::arrow::{array::ArrayRef, ffi}; +use polars_core::utils::arrow::ffi; use polars_core::POOL; use pyo3::ffi::Py_uintptr_t; use pyo3::prelude::*; diff --git a/py-polars/src/dataframe.rs b/py-polars/src/dataframe.rs index b32c3f4f384b..1458e5ab08f2 100644 --- a/py-polars/src/dataframe.rs +++ b/py-polars/src/dataframe.rs @@ -653,7 +653,6 @@ impl PyDataFrame { .map(|(i, _)| i) .collect::>(); - use polars_core::export::arrow::array::ArrayRef; let rbs = self .df .iter_chunks() @@ -671,7 +670,6 @@ impl PyDataFrame { CastOptions::default(), ) .unwrap(); - let out = Arc::from(out) as ArrayRef; *arr = out; } let rb = ArrowChunk::new(rb);