diff --git a/crates/polars-arrow/src/array/binview/mutable.rs b/crates/polars-arrow/src/array/binview/mutable.rs index e676ffcb7aa8..ec29bd2b60e3 100644 --- a/crates/polars-arrow/src/array/binview/mutable.rs +++ b/crates/polars-arrow/src/array/binview/mutable.rs @@ -152,6 +152,7 @@ impl MutableBinaryViewArray { } } + #[inline] pub fn push_value_ignore_validity>(&mut self, value: V) { let value = value.as_ref(); let bytes = value.to_bytes(); @@ -187,6 +188,7 @@ impl MutableBinaryViewArray { self.views.push(value); } + #[inline] pub fn push_value>(&mut self, value: V) { if let Some(validity) = &mut self.validity { validity.push(true) @@ -194,6 +196,7 @@ impl MutableBinaryViewArray { self.push_value_ignore_validity(value) } + #[inline] pub fn push>(&mut self, value: Option) { if let Some(value) = value { self.push_value(value) @@ -202,6 +205,7 @@ impl MutableBinaryViewArray { } } + #[inline] pub fn push_null(&mut self) { self.views.push(View::default()); match &mut self.validity { diff --git a/crates/polars-arrow/src/array/boolean/mutable.rs b/crates/polars-arrow/src/array/boolean/mutable.rs index 9af94cc9dfa7..80d689806f1d 100644 --- a/crates/polars-arrow/src/array/boolean/mutable.rs +++ b/crates/polars-arrow/src/array/boolean/mutable.rs @@ -98,23 +98,30 @@ impl MutableBooleanArray { } } + #[inline] + pub fn push_value(&mut self, value: bool) { + self.values.push(value); + match &mut self.validity { + Some(validity) => validity.push(true), + None => {}, + } + } + + #[inline] + pub fn push_null(&mut self) { + self.values.push(false); + match &mut self.validity { + Some(validity) => validity.push(false), + None => self.init_validity(), + } + } + /// Pushes a new entry to [`MutableBooleanArray`]. + #[inline] pub fn push(&mut self, value: Option) { match value { - Some(value) => { - self.values.push(value); - match &mut self.validity { - Some(validity) => validity.push(true), - None => {}, - } - }, - None => { - self.values.push(false); - match &mut self.validity { - Some(validity) => validity.push(false), - None => self.init_validity(), - } - }, + Some(value) => self.push_value(value), + None => self.push_null(), } } @@ -232,6 +239,10 @@ impl MutableBooleanArray { let a: BooleanArray = self.into(); Arc::new(a) } + + pub fn freeze(self) -> BooleanArray { + self.into() + } } /// Getters diff --git a/crates/polars-arrow/src/pushable.rs b/crates/polars-arrow/src/pushable.rs index 12d04ebdcf76..025e78275dcd 100644 --- a/crates/polars-arrow/src/pushable.rs +++ b/crates/polars-arrow/src/pushable.rs @@ -1,10 +1,15 @@ -use crate::array::{MutableBinaryViewArray, MutablePrimitiveArray, ViewType}; -use crate::bitmap::MutableBitmap; -use crate::offset::{Offset, Offsets}; +use crate::array::{ + BinaryViewArrayGeneric, BooleanArray, MutableBinaryViewArray, MutableBooleanArray, + MutablePrimitiveArray, PrimitiveArray, ViewType, +}; +use crate::bitmap::{Bitmap, MutableBitmap}; +use crate::offset::{Offset, Offsets, OffsetsBuffer}; use crate::types::NativeType; /// A private trait representing structs that can receive elements. pub trait Pushable: Sized + Default { + type Freeze; + fn with_capacity(capacity: usize) -> Self { let mut new = Self::default(); new.reserve(capacity); @@ -16,9 +21,12 @@ pub trait Pushable: Sized + Default { fn push_null(&mut self); fn extend_constant(&mut self, additional: usize, value: T); fn extend_null_constant(&mut self, additional: usize); + fn freeze(self) -> Self::Freeze; } impl Pushable for MutableBitmap { + type Freeze = Bitmap; + #[inline] fn reserve(&mut self, additional: usize) { MutableBitmap::reserve(self, additional) @@ -47,9 +55,14 @@ impl Pushable for MutableBitmap { fn extend_null_constant(&mut self, additional: usize) { self.extend_constant(additional, false) } + + fn freeze(self) -> Self::Freeze { + self.into() + } } impl Pushable for Vec { + type Freeze = Vec; #[inline] fn reserve(&mut self, additional: usize) { Vec::reserve(self, additional) @@ -78,8 +91,12 @@ impl Pushable for Vec { fn extend_null_constant(&mut self, additional: usize) { self.extend_constant(additional, T::default()) } + fn freeze(self) -> Self::Freeze { + self + } } impl Pushable for Offsets { + type Freeze = OffsetsBuffer; fn reserve(&mut self, additional: usize) { self.reserve(additional) } @@ -107,9 +124,14 @@ impl Pushable for Offsets { fn extend_null_constant(&mut self, additional: usize) { self.extend_constant(additional) } + fn freeze(self) -> Self::Freeze { + self.into() + } } impl Pushable> for MutablePrimitiveArray { + type Freeze = PrimitiveArray; + #[inline] fn reserve(&mut self, additional: usize) { MutablePrimitiveArray::reserve(self, additional) @@ -139,16 +161,29 @@ impl Pushable> for MutablePrimitiveArray { fn extend_null_constant(&mut self, additional: usize) { MutablePrimitiveArray::extend_constant(self, additional, None) } + fn freeze(self) -> Self::Freeze { + self.into() + } } -impl Pushable<&T> for MutableBinaryViewArray { +pub trait NoOption {} +impl NoOption for &str {} +impl NoOption for &[u8] {} + +impl Pushable for MutableBinaryViewArray +where + T: AsRef + NoOption, + K: ViewType + ?Sized, +{ + type Freeze = BinaryViewArrayGeneric; + #[inline] fn reserve(&mut self, additional: usize) { MutableBinaryViewArray::reserve(self, additional) } #[inline] - fn push(&mut self, value: &T) { + fn push(&mut self, value: T) { MutableBinaryViewArray::push_value(self, value) } @@ -161,7 +196,8 @@ impl Pushable<&T> for MutableBinaryViewArray { MutableBinaryViewArray::push_null(self) } - fn extend_constant(&mut self, additional: usize, value: &T) { + fn extend_constant(&mut self, additional: usize, value: T) { + let value = value.as_ref(); // First push a value to get the View MutableBinaryViewArray::push_value(self, value); @@ -183,4 +219,132 @@ impl Pushable<&T> for MutableBinaryViewArray { fn extend_null_constant(&mut self, additional: usize) { self.extend_null(additional); } + fn freeze(self) -> Self::Freeze { + self.into() + } +} + +impl Pushable> for MutableBinaryViewArray +where + T: AsRef, + K: ViewType + ?Sized, +{ + type Freeze = BinaryViewArrayGeneric; + #[inline] + fn reserve(&mut self, additional: usize) { + MutableBinaryViewArray::reserve(self, additional) + } + + #[inline] + fn push(&mut self, value: Option) { + MutableBinaryViewArray::push(self, value.as_ref()) + } + + #[inline] + fn len(&self) -> usize { + MutableBinaryViewArray::len(self) + } + + fn push_null(&mut self) { + MutableBinaryViewArray::push_null(self) + } + + fn extend_constant(&mut self, additional: usize, value: Option) { + let value = value.as_ref(); + // First push a value to get the View + MutableBinaryViewArray::push(self, value); + + // And then use that new view to extend + let views = self.views_mut(); + let view = *views.last().unwrap(); + + let remaining = additional - 1; + for _ in 0..remaining { + views.push(view); + } + + if let Some(bitmap) = self.validity() { + bitmap.extend_constant(remaining, true) + } + } + + #[inline] + fn extend_null_constant(&mut self, additional: usize) { + self.extend_null(additional); + } + fn freeze(self) -> Self::Freeze { + self.into() + } +} + +impl Pushable for MutableBooleanArray { + type Freeze = BooleanArray; + #[inline] + fn reserve(&mut self, additional: usize) { + MutableBooleanArray::reserve(self, additional) + } + + #[inline] + fn push(&mut self, value: bool) { + MutableBooleanArray::push_value(self, value) + } + + #[inline] + fn len(&self) -> usize { + self.values().len() + } + + #[inline] + fn push_null(&mut self) { + unimplemented!() + } + + #[inline] + fn extend_constant(&mut self, additional: usize, value: bool) { + MutableBooleanArray::extend_constant(self, additional, Some(value)) + } + + #[inline] + fn extend_null_constant(&mut self, _additional: usize) { + unimplemented!() + } + fn freeze(self) -> Self::Freeze { + self.into() + } +} + +impl Pushable> for MutableBooleanArray { + type Freeze = BooleanArray; + #[inline] + fn reserve(&mut self, additional: usize) { + MutableBooleanArray::reserve(self, additional) + } + + #[inline] + fn push(&mut self, value: Option) { + MutableBooleanArray::push(self, value) + } + + #[inline] + fn len(&self) -> usize { + self.values().len() + } + + #[inline] + fn push_null(&mut self) { + MutableBooleanArray::push_null(self) + } + + #[inline] + fn extend_constant(&mut self, additional: usize, value: Option) { + MutableBooleanArray::extend_constant(self, additional, value) + } + + #[inline] + fn extend_null_constant(&mut self, additional: usize) { + MutableBooleanArray::extend_constant(self, additional, None) + } + fn freeze(self) -> Self::Freeze { + self.into() + } } diff --git a/crates/polars-core/src/chunked_array/builder/boolean.rs b/crates/polars-core/src/chunked_array/builder/boolean.rs index 407bc3abcf53..465a63593662 100644 --- a/crates/polars-core/src/chunked_array/builder/boolean.rs +++ b/crates/polars-core/src/chunked_array/builder/boolean.rs @@ -10,13 +10,13 @@ impl ChunkedBuilder for BooleanChunkedBuilder { /// Appends a value of type `T` into the builder #[inline] fn append_value(&mut self, v: bool) { - self.array_builder.push(Some(v)); + self.array_builder.push_value(v); } /// Appends a null slot into the builder #[inline] fn append_null(&mut self) { - self.array_builder.push(None); + self.array_builder.push_null(); } fn finish(mut self) -> BooleanChunked { diff --git a/crates/polars-core/src/chunked_array/from.rs b/crates/polars-core/src/chunked_array/from.rs index b65b4b71d9a8..ab52e4ef69ff 100644 --- a/crates/polars-core/src/chunked_array/from.rs +++ b/crates/polars-core/src/chunked_array/from.rs @@ -331,3 +331,31 @@ impl BooleanChunked { Self::with_chunk(name, arr) } } + +impl<'a, T> From<&'a ChunkedArray> for Vec>> +where + T: PolarsDataType, +{ + fn from(ca: &'a ChunkedArray) -> Self { + let mut out = Vec::with_capacity(ca.len()); + for arr in ca.downcast_iter() { + out.extend(arr.iter()) + } + out + } +} +impl From for Vec> { + fn from(ca: StringChunked) -> Self { + ca.iter().map(|opt| opt.map(|s| s.to_string())).collect() + } +} + +impl From for Vec> { + fn from(ca: BooleanChunked) -> Self { + let mut out = Vec::with_capacity(ca.len()); + for arr in ca.downcast_iter() { + out.extend(arr.iter()) + } + out + } +} diff --git a/crates/polars-core/src/chunked_array/from_iterator_par.rs b/crates/polars-core/src/chunked_array/from_iterator_par.rs index a8bd27c7048d..88b135ad0405 100644 --- a/crates/polars-core/src/chunked_array/from_iterator_par.rs +++ b/crates/polars-core/src/chunked_array/from_iterator_par.rs @@ -1,8 +1,7 @@ //! Implementations of upstream traits for [`ChunkedArray`] use std::collections::LinkedList; -use arrow::bitmap::{Bitmap, MutableBitmap}; -use polars_utils::sync::SyncPtr; +use arrow::pushable::{NoOption, Pushable}; use rayon::prelude::*; use super::from_iterator::PolarsAsRef; @@ -29,7 +28,7 @@ fn list_append(mut list1: LinkedList, mut list2: LinkedList) -> LinkedL list1 } -fn collect_into_linked_list(par_iter: I) -> LinkedList> +fn collect_into_linked_list_vec(par_iter: I) -> LinkedList> where I: IntoParallelIterator, { @@ -42,21 +41,25 @@ where .reduce(LinkedList::new, list_append) } -fn get_capacity_from_par_results(ll: &LinkedList>) -> usize { - ll.iter().map(|list| list.len()).sum() +fn collect_into_linked_list(par_iter: I, identity: F) -> LinkedList +where + I: IntoParallelIterator, + P: Pushable + Send + Sync, + F: Fn() -> P + Sync + Send, + P::Freeze: Send, +{ + let it = par_iter.into_par_iter(); + it.fold(identity, |mut v, item| { + v.push(item); + v + }) + // The freeze on this line, ensures the null count is done in parallel + .map(|p| as_list(p.freeze())) + .reduce(LinkedList::new, list_append) } -fn get_capacity_from_par_results_slice(bufs: &[Vec]) -> usize { - bufs.iter().map(|list| list.len()).sum() -} -fn get_offsets(bufs: &[Vec]) -> Vec { - bufs.iter() - .scan(0usize, |acc, buf| { - let out = *acc; - *acc += buf.len(); - Some(out) - }) - .collect() +fn get_capacity_from_par_results(ll: &LinkedList>) -> usize { + ll.iter().map(|list| list.len()).sum() } impl FromParallelIterator for NoNull> @@ -65,155 +68,54 @@ where { fn from_par_iter>(iter: I) -> Self { // Get linkedlist filled with different vec result from different threads - let vectors = collect_into_linked_list(iter); + let vectors = collect_into_linked_list_vec(iter); let vectors = vectors.into_iter().collect::>(); let values = flatten_par(&vectors); NoNull::new(ChunkedArray::new_vec("", values)) } } -fn finish_validities(validities: Vec<(Option, usize)>, capacity: usize) -> Option { - if validities.iter().any(|(v, _)| v.is_some()) { - let mut bitmap = MutableBitmap::with_capacity(capacity); - for (valids, len) in validities { - if let Some(valids) = valids { - bitmap.extend_from_bitmap(&(valids)) - } else { - bitmap.extend_constant(len, true) - } - } - Some(bitmap.into()) - } else { - None - } -} - impl FromParallelIterator> for ChunkedArray where T: PolarsNumericType, { fn from_par_iter>>(iter: I) -> Self { - // Get linkedlist filled with different vec result from different threads - let vectors = collect_into_linked_list(iter); - - let vectors = vectors.into_iter().collect::>(); - let capacity: usize = get_capacity_from_par_results_slice(&vectors); - let offsets = get_offsets(&vectors); - - let mut values_buf: Vec = Vec::with_capacity(capacity); - let values_buf_ptr = unsafe { SyncPtr::new(values_buf.as_mut_ptr()) }; - - let validities = offsets - .into_par_iter() - .zip(vectors) - .map(|(offset, vector)| { - let mut local_validity = None; - let local_len = vector.len(); - let mut latest_validy_written = 0; - unsafe { - let values_buf_ptr = values_buf_ptr.get().add(offset); - - for (i, opt_v) in vector.into_iter().enumerate() { - match opt_v { - Some(v) => { - std::ptr::write(values_buf_ptr.add(i), v); - }, - None => { - let validity = match &mut local_validity { - None => { - let validity = MutableBitmap::with_capacity(local_len); - local_validity = Some(validity); - local_validity.as_mut().unwrap_unchecked() - }, - Some(validity) => validity, - }; - validity.extend_constant(i - latest_validy_written, true); - latest_validy_written = i + 1; - validity.push_unchecked(false); - // initialize value - std::ptr::write(values_buf_ptr.add(i), T::Native::default()); - }, - } - } - } - if let Some(validity) = &mut local_validity { - validity.extend_constant(local_len - latest_validy_written, true); - } - (local_validity.map(|b| b.into()), local_len) - }) - .collect::>(); - unsafe { values_buf.set_len(capacity) }; - - let validity = finish_validities(validities, capacity); - - let arr = PrimitiveArray::from_data_default(values_buf.into(), validity); - arr.into() + let chunks = collect_into_linked_list(iter, MutablePrimitiveArray::new); + Self::from_chunk_iter("", chunks) } } impl FromParallelIterator for BooleanChunked { fn from_par_iter>(iter: I) -> Self { - let vectors = collect_into_linked_list(iter); - - let capacity: usize = get_capacity_from_par_results(&vectors); - - let arr = unsafe { - BooleanArray::from_trusted_len_values_iter( - vectors.into_iter().flatten().trust_my_length(capacity), - ) - }; - arr.into() + let chunks = collect_into_linked_list(iter, MutableBooleanArray::new); + Self::from_chunk_iter("", chunks) } } impl FromParallelIterator> for BooleanChunked { fn from_par_iter>>(iter: I) -> Self { - // Get linkedlist filled with different vec result from different threads. - let vectors = collect_into_linked_list(iter); - let vectors = vectors.into_iter().collect::>(); - let chunks: Vec = vectors - .into_par_iter() - .map(|vector| vector.into()) - .collect(); - Self::from_chunk_iter("", chunks).rechunk() + let chunks = collect_into_linked_list(iter, MutableBooleanArray::new); + Self::from_chunk_iter("", chunks) } } impl FromParallelIterator for StringChunked where - Ptr: PolarsAsRef + Send + Sync, + Ptr: PolarsAsRef + Send + Sync + NoOption, { fn from_par_iter>(iter: I) -> Self { - let vectors = collect_into_linked_list(iter); - let cap = get_capacity_from_par_results(&vectors); - - let mut builder = MutableBinaryViewArray::with_capacity(cap); - // TODO! we can do this in parallel ind just combine the buffers. - for vec in vectors { - for val in vec { - builder.push_value_ignore_validity(val.as_ref()) - } - } - ChunkedArray::with_chunk("", builder.freeze()) + let chunks = collect_into_linked_list(iter, MutableBinaryViewArray::new); + Self::from_chunk_iter("", chunks) } } impl FromParallelIterator for BinaryChunked where - Ptr: PolarsAsRef<[u8]> + Send + Sync, + Ptr: PolarsAsRef<[u8]> + Send + Sync + NoOption, { fn from_par_iter>(iter: I) -> Self { - let vectors = collect_into_linked_list(iter); - let cap = get_capacity_from_par_results(&vectors); - - let mut builder = MutableBinaryViewArray::with_capacity(cap); - // TODO! we can do this in parallel ind just combine the buffers. - for vec in vectors { - for val in vec { - builder.push_value_ignore_validity(val.as_ref()) - } - } - ChunkedArray::with_chunk("", builder.freeze()) + let chunks = collect_into_linked_list(iter, MutableBinaryViewArray::new); + Self::from_chunk_iter("", chunks) } } @@ -222,29 +124,8 @@ where Ptr: AsRef + Send + Sync, { fn from_par_iter>>(iter: I) -> Self { - let vectors = collect_into_linked_list(iter); - let vectors = vectors.into_iter().collect::>(); - - let arrays = vectors - .into_par_iter() - .map(|vector| { - let cap = vector.len(); - let mut mutable = MutableBinaryViewArray::with_capacity(cap); - for opt_val in vector { - mutable.push(opt_val) - } - mutable.freeze() - }) - .collect::>(); - - // TODO! - // do this in parallel. - let arrays = arrays - .iter() - .map(|arr| arr as &dyn Array) - .collect::>(); - let arr = arrow::compute::concatenate::concatenate(&arrays).unwrap(); - unsafe { StringChunked::from_chunks("", vec![arr]) } + let chunks = collect_into_linked_list(iter, MutableBinaryViewArray::new); + Self::from_chunk_iter("", chunks) } } @@ -253,57 +134,8 @@ where Ptr: AsRef<[u8]> + Send + Sync, { fn from_par_iter>>(iter: I) -> Self { - let vectors = collect_into_linked_list(iter); - let vectors = vectors.into_iter().collect::>(); - - let arrays = vectors - .into_par_iter() - .map(|vector| { - let cap = vector.len(); - let mut mutable = MutableBinaryViewArray::with_capacity(cap); - for opt_val in vector { - mutable.push(opt_val) - } - mutable.freeze() - }) - .collect::>(); - - // TODO! - // do this in parallel. - let arrays = arrays - .iter() - .map(|arr| arr as &dyn Array) - .collect::>(); - let arr = arrow::compute::concatenate::concatenate(&arrays).unwrap(); - unsafe { BinaryChunked::from_chunks("", vec![arr]) } - } -} - -impl<'a, T> From<&'a ChunkedArray> for Vec>> -where - T: PolarsDataType, -{ - fn from(ca: &'a ChunkedArray) -> Self { - let mut out = Vec::with_capacity(ca.len()); - for arr in ca.downcast_iter() { - out.extend(arr.iter()) - } - out - } -} -impl From for Vec> { - fn from(ca: StringChunked) -> Self { - ca.iter().map(|opt| opt.map(|s| s.to_string())).collect() - } -} - -impl From for Vec> { - fn from(ca: BooleanChunked) -> Self { - let mut out = Vec::with_capacity(ca.len()); - for arr in ca.downcast_iter() { - out.extend(arr.iter()) - } - out + let chunks = collect_into_linked_list(iter, MutableBinaryViewArray::new); + Self::from_chunk_iter("", chunks) } } @@ -314,7 +146,7 @@ impl FromParallelIterator> for ListChunked { I: IntoParallelIterator>, { let mut dtype = None; - let vectors = collect_into_linked_list(iter); + let vectors = collect_into_linked_list_vec(iter); let list_capacity: usize = get_capacity_from_par_results(&vectors); let value_capacity = vectors diff --git a/crates/polars-lazy/src/physical_plan/expressions/group_iter.rs b/crates/polars-lazy/src/physical_plan/expressions/group_iter.rs index 3949b7c37961..5a1fb12d7865 100644 --- a/crates/polars-lazy/src/physical_plan/expressions/group_iter.rs +++ b/crates/polars-lazy/src/physical_plan/expressions/group_iter.rs @@ -34,7 +34,7 @@ impl<'a> AggregationContext<'a> { // SAFETY: dtype is correct unsafe { Box::new(FlatIter::new( - s.array_ref(0).clone(), + s.chunks(), self.groups.len(), s.dtype(), name, @@ -107,8 +107,10 @@ impl<'a> Iterator for LitIter<'a> { } struct FlatIter<'a> { - array: ArrayRef, + current_array: ArrayRef, + chunks: Vec, offset: usize, + chunk_offset: usize, len: usize, // UnstableSeries referenced that series #[allow(dead_code)] @@ -119,16 +121,23 @@ struct FlatIter<'a> { impl<'a> FlatIter<'a> { /// # Safety /// Caller must ensure the given `logical` dtype belongs to `array`. - unsafe fn new(array: ArrayRef, len: usize, logical: &DataType, name: &str) -> Self { + unsafe fn new(chunks: &[ArrayRef], len: usize, logical: &DataType, name: &str) -> Self { + let mut stack = Vec::with_capacity(chunks.len()); + for chunk in chunks.iter().rev() { + stack.push(chunk.clone()) + } + let current_array = stack.pop().unwrap(); let mut series_container = Box::pin(Series::from_chunks_and_dtype_unchecked( name, - vec![array.clone()], + vec![current_array.clone()], logical, )); let ref_s = &mut *series_container as *mut Series; Self { - array, + current_array, + chunks: stack, offset: 0, + chunk_offset: 0, len, series_container, // SAFETY: we pinned the series so the location is still valid @@ -144,9 +153,21 @@ impl<'a> Iterator for FlatIter<'a> { if self.len == self.offset { None } else { - let mut arr = unsafe { self.array.sliced_unchecked(self.offset, 1) }; + if self.chunk_offset < self.current_array.len() { + let mut arr = unsafe { self.current_array.sliced_unchecked(self.chunk_offset, 1) }; + self.item.swap(&mut arr); + } else { + match self.chunks.pop() { + Some(arr) => { + self.current_array = arr; + self.chunk_offset = 0; + return self.next(); + }, + None => return None, + } + } self.offset += 1; - self.item.swap(&mut arr); + self.chunk_offset += 1; Some(Some(self.item)) } } diff --git a/crates/polars-parquet/src/arrow/read/deserialize/binary/utils.rs b/crates/polars-parquet/src/arrow/read/deserialize/binary/utils.rs index 11a16351ea45..b501277e78fc 100644 --- a/crates/polars-parquet/src/arrow/read/deserialize/binary/utils.rs +++ b/crates/polars-parquet/src/arrow/read/deserialize/binary/utils.rs @@ -54,6 +54,7 @@ impl Binary { } impl<'a, O: Offset> Pushable<&'a [u8]> for Binary { + type Freeze = (); #[inline] fn reserve(&mut self, additional: usize) { let avg_len = self.values.len() / std::cmp::max(self.offsets.last().to_usize(), 1); @@ -85,6 +86,9 @@ impl<'a, O: Offset> Pushable<&'a [u8]> for Binary { fn extend_null_constant(&mut self, additional: usize) { self.extend_constant(additional) } + fn freeze(self) -> Self::Freeze { + unimplemented!() + } } #[derive(Debug)] diff --git a/crates/polars-parquet/src/arrow/read/deserialize/fixed_size_binary/utils.rs b/crates/polars-parquet/src/arrow/read/deserialize/fixed_size_binary/utils.rs index 50a89442570a..912dfb9b0ac0 100644 --- a/crates/polars-parquet/src/arrow/read/deserialize/fixed_size_binary/utils.rs +++ b/crates/polars-parquet/src/arrow/read/deserialize/fixed_size_binary/utils.rs @@ -30,6 +30,7 @@ impl FixedSizeBinary { } impl<'a> Pushable<&'a [u8]> for FixedSizeBinary { + type Freeze = (); #[inline] fn reserve(&mut self, additional: usize) { self.values.reserve(additional * self.size); @@ -60,4 +61,7 @@ impl<'a> Pushable<&'a [u8]> for FixedSizeBinary { fn extend_null_constant(&mut self, additional: usize) { self.extend_constant(additional) } + fn freeze(self) -> Self::Freeze { + unimplemented!() + } } diff --git a/crates/polars/tests/it/arrow/array/boolean/mutable.rs b/crates/polars/tests/it/arrow/array/boolean/mutable.rs index 1071a1ed8c37..5264c552edab 100644 --- a/crates/polars/tests/it/arrow/array/boolean/mutable.rs +++ b/crates/polars/tests/it/arrow/array/boolean/mutable.rs @@ -19,8 +19,8 @@ fn set() { #[test] fn push() { let mut a = MutableBooleanArray::new(); - a.push(Some(true)); - a.push(Some(false)); + a.push_value(true); + a.push_value(false); a.push(None); a.push_null(); assert_eq!(