From cc65d6ad36e3997d9720d3bb09a7cea71cf230ce Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Thu, 2 Apr 2026 16:24:02 -0400 Subject: [PATCH 01/15] Follow up Signed-off-by: Nicholas Gates --- encodings/bytebool/src/array.rs | 79 +++--- encodings/bytebool/src/compute.rs | 11 +- encodings/bytebool/src/slice.rs | 3 +- encodings/datetime-parts/src/array.rs | 121 +++------ encodings/datetime-parts/src/compute/cast.rs | 3 +- .../datetime-parts/src/compute/filter.rs | 3 +- encodings/datetime-parts/src/compute/mask.rs | 3 +- encodings/datetime-parts/src/compute/rules.rs | 3 +- encodings/datetime-parts/src/compute/slice.rs | 11 +- encodings/datetime-parts/src/compute/take.rs | 12 +- .../src/decimal_byte_parts/compute/cast.rs | 7 +- .../src/decimal_byte_parts/compute/compare.rs | 2 +- .../src/decimal_byte_parts/compute/filter.rs | 11 +- .../src/decimal_byte_parts/compute/mask.rs | 11 +- .../src/decimal_byte_parts/compute/take.rs | 12 +- .../src/decimal_byte_parts/mod.rs | 112 +++----- .../src/decimal_byte_parts/rules.rs | 12 +- .../src/decimal_byte_parts/slice.rs | 17 +- encodings/pco/src/array.rs | 137 +++++----- encodings/pco/src/compute/cast.rs | 20 +- encodings/pco/src/slice.rs | 4 +- encodings/runend/src/array.rs | 113 ++++---- encodings/runend/src/compute/cast.rs | 3 +- encodings/runend/src/compute/fill_null.rs | 3 +- encodings/runend/src/compute/filter.rs | 3 +- encodings/runend/src/kernel.rs | 7 +- encodings/runend/src/rules.rs | 3 +- encodings/sequence/src/array.rs | 241 ++++++++---------- encodings/sequence/src/compress.rs | 7 +- encodings/sequence/src/compute/cast.rs | 5 +- encodings/sequence/src/compute/min_max.rs | 4 +- encodings/sequence/src/compute/slice.rs | 3 +- encodings/sparse/src/canonical.rs | 13 +- encodings/sparse/src/compute/cast.rs | 3 +- encodings/sparse/src/compute/filter.rs | 3 +- encodings/sparse/src/compute/take.rs | 3 +- encodings/sparse/src/lib.rs | 140 +++++----- encodings/sparse/src/rules.rs | 5 +- encodings/sparse/src/slice.rs | 3 +- encodings/zigzag/src/array.rs | 74 ++---- encodings/zigzag/src/compress.rs | 4 +- encodings/zigzag/src/compute/cast.rs | 3 +- encodings/zigzag/src/compute/mod.rs | 7 +- encodings/zigzag/src/slice.rs | 5 +- vortex-array/src/array/erased.rs | 5 - vortex-array/src/array/mod.rs | 14 +- vortex-array/src/array/typed.rs | 64 ++++- vortex-array/src/array/vtable/dyn_.rs | 37 +-- vortex-array/src/array/vtable/mod.rs | 84 ++---- vortex-array/src/arrays/bool/array.rs | 16 +- vortex-array/src/arrays/bool/vtable/mod.rs | 46 +--- vortex-array/src/arrays/chunked/array.rs | 26 +- vortex-array/src/arrays/chunked/vtable/mod.rs | 57 ++--- .../src/arrays/chunked/vtable/validity.rs | 4 +- vortex-array/src/arrays/constant/array.rs | 6 +- .../src/arrays/constant/compute/take.rs | 4 +- .../src/arrays/constant/vtable/mod.rs | 43 +--- vortex-array/src/arrays/decimal/array.rs | 43 +++- vortex-array/src/arrays/decimal/vtable/mod.rs | 54 +--- vortex-array/src/arrays/dict/array.rs | 24 +- vortex-array/src/arrays/dict/vtable/mod.rs | 50 +--- .../src/arrays/dict/vtable/validity.rs | 13 +- vortex-array/src/arrays/extension/array.rs | 11 +- .../src/arrays/extension/vtable/mod.rs | 40 +-- vortex-array/src/arrays/filter/array.rs | 11 +- vortex-array/src/arrays/filter/vtable.rs | 42 +-- .../src/arrays/fixed_size_list/array.rs | 20 +- .../src/arrays/fixed_size_list/vtable/mod.rs | 42 +-- vortex-array/src/arrays/list/array.rs | 21 +- vortex-array/src/arrays/list/vtable/mod.rs | 50 +--- vortex-array/src/arrays/listview/array.rs | 23 +- .../src/arrays/listview/conversion.rs | 8 +- .../src/arrays/listview/vtable/mod.rs | 52 +--- vortex-array/src/arrays/masked/array.rs | 6 +- .../src/arrays/masked/compute/filter.rs | 6 +- .../src/arrays/masked/compute/slice.rs | 4 +- .../src/arrays/masked/compute/take.rs | 6 +- vortex-array/src/arrays/masked/vtable/mod.rs | 44 +--- vortex-array/src/arrays/null/mod.rs | 42 +-- vortex-array/src/arrays/patched/array.rs | 20 ++ vortex-array/src/arrays/patched/vtable/mod.rs | 90 +++---- .../src/arrays/primitive/array/mod.rs | 45 +++- .../src/arrays/primitive/vtable/mod.rs | 40 +-- vortex-array/src/arrays/scalar_fn/array.rs | 7 +- vortex-array/src/arrays/scalar_fn/rules.rs | 3 +- .../src/arrays/scalar_fn/vtable/mod.rs | 81 +----- vortex-array/src/arrays/shared/array.rs | 6 +- vortex-array/src/arrays/shared/vtable.rs | 41 +-- vortex-array/src/arrays/slice/array.rs | 11 +- vortex-array/src/arrays/slice/vtable.rs | 42 +-- vortex-array/src/arrays/struct_/array.rs | 43 +++- vortex-array/src/arrays/struct_/vtable/mod.rs | 39 +-- vortex-array/src/arrays/varbin/array.rs | 106 ++++++-- vortex-array/src/arrays/varbin/builder.rs | 4 +- vortex-array/src/arrays/varbin/vtable/mod.rs | 46 +--- vortex-array/src/arrays/varbinview/array.rs | 70 +++-- .../src/arrays/varbinview/vtable/mod.rs | 39 +-- vortex-array/src/arrays/variant/mod.rs | 7 +- vortex-array/src/arrays/variant/vtable/mod.rs | 41 +-- .../src/display/extractors/metadata.rs | 7 +- vortex-array/src/scalar_fn/fns/select.rs | 9 +- 101 files changed, 1293 insertions(+), 1771 deletions(-) diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index 04744355a11..2e04a966943 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -4,6 +4,7 @@ use std::fmt::Debug; use vortex_array::Array; +use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; @@ -19,7 +20,6 @@ use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; @@ -44,28 +44,15 @@ vtable!(ByteBool, ByteBool, ByteBoolData); impl VTable for ByteBool { type ArrayData = ByteBoolData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromValidityHelper; - fn vtable(_array: &Self::ArrayData) -> &Self { - &ByteBool - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ByteBoolData) -> usize { - array.buffer.len() - } - - fn dtype(array: &ByteBoolData) -> &DType { - &array.dtype - } - - fn stats(array: &ByteBoolData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + ByteBoolData::validate(data.buffer(), data.validity(), dtype, len) } fn array_hash(array: &ByteBoolData, state: &mut H, precision: Precision) { @@ -96,31 +83,20 @@ impl VTable for ByteBool { } } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) - } - - fn build( + &self, dtype: &DType, len: usize, - _metadata: &Self::Metadata, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + ::deserialize(metadata)?; let validity = if children.is_empty() { Validity::from(dtype.nullability()) } else if children.len() == 1 { @@ -155,7 +131,7 @@ impl VTable for ByteBool { ); array.validity = match &slots[VALIDITY_SLOT] { Some(arr) => Validity::Array(arr.clone()), - None => Validity::from(array.dtype.nullability()), + None => Validity::from(array.validity.nullability()), }; array.slots = slots; Ok(()) @@ -194,11 +170,9 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; #[derive(Clone, Debug)] pub struct ByteBoolData { - dtype: DType, buffer: BufferHandle, validity: Validity, pub(super) slots: Vec>, - stats_set: ArrayStats, } #[derive(Clone, Debug)] @@ -207,14 +181,40 @@ pub struct ByteBool; impl ByteBool { pub const ID: ArrayId = ArrayId::new_ref("vortex.bytebool"); + pub fn new(buffer: BufferHandle, validity: Validity) -> ByteBoolArray { + let dtype = DType::Bool(validity.nullability()); + let data = ByteBoolData::new(buffer, validity); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(ByteBool, dtype, len, data)) + .vortex_expect("ByteBoolData is always valid") + } + /// Construct a [`ByteBoolArray`] from a `Vec` and validity. pub fn from_vec>(data: Vec, validity: V) -> ByteBoolArray { - Array::try_from_data(ByteBoolData::from_vec(data, validity)) + let data = ByteBoolData::from_vec(data, validity); + let dtype = DType::Bool(data.validity.nullability()); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(ByteBool, dtype, len, data)) .vortex_expect("ByteBoolData is always valid") } } impl ByteBoolData { + pub fn validate( + buffer: &BufferHandle, + validity: &Validity, + dtype: &DType, + len: usize, + ) -> VortexResult<()> { + let expected_dtype = DType::Bool(validity.nullability()); + vortex_ensure!(dtype == &expected_dtype, "expected dtype {expected_dtype}, got {dtype}"); + vortex_ensure!(buffer.len() == len, "expected len {len}, got {}", buffer.len()); + if let Some(vlen) = validity.maybe_len() { + vortex_ensure!(vlen == len, "expected validity len {len}, got {vlen}"); + } + Ok(()) + } + fn make_slots(validity: &Validity, len: usize) -> Vec> { vec![validity_to_child(validity, len)] } @@ -232,11 +232,9 @@ impl ByteBoolData { } let slots = Self::make_slots(&validity, length); Self { - dtype: DType::Bool(validity.nullability()), buffer, validity, slots, - stats_set: Default::default(), } } @@ -250,11 +248,6 @@ impl ByteBoolData { self.buffer.len() == 0 } - /// Returns the logical data type of the array. - pub fn dtype(&self) -> &DType { - &self.dtype - } - /// Returns the validity mask for this array. pub fn validity_mask(&self) -> Mask { self.validity.to_mask(self.len()) diff --git a/encodings/bytebool/src/compute.rs b/encodings/bytebool/src/compute.rs index a44f30cce0c..609a83f799a 100644 --- a/encodings/bytebool/src/compute.rs +++ b/encodings/bytebool/src/compute.rs @@ -16,7 +16,6 @@ use vortex_array::validity::Validity; use vortex_error::VortexResult; use super::ByteBool; -use super::ByteBoolData; impl CastReduce for ByteBool { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { @@ -31,9 +30,7 @@ impl CastReduce for ByteBool { .clone() .cast_nullability(dtype.nullability(), array.len())?; - return Ok(Some( - ByteBoolData::new(array.buffer().clone(), new_validity).into_array(), - )); + return Ok(Some(ByteBool::new(array.buffer().clone(), new_validity).into_array())); } // For other casts, decode to canonical and let BoolArray handle it @@ -44,7 +41,7 @@ impl CastReduce for ByteBool { impl MaskReduce for ByteBool { fn mask(array: ArrayView<'_, Self>, mask: &ArrayRef) -> VortexResult> { Ok(Some( - ByteBoolData::new( + ByteBool::new( array.buffer().clone(), array .validity() @@ -79,9 +76,7 @@ impl TakeExecute for ByteBool { .collect::>() }); - Ok(Some( - ByteBoolData::from_vec(taken_bools, validity).into_array(), - )) + Ok(Some(ByteBool::from_vec(taken_bools, validity).into_array())) } } diff --git a/encodings/bytebool/src/slice.rs b/encodings/bytebool/src/slice.rs index b5926602808..4c55f5bbd40 100644 --- a/encodings/bytebool/src/slice.rs +++ b/encodings/bytebool/src/slice.rs @@ -10,12 +10,11 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::ByteBool; -use crate::ByteBoolData; impl SliceReduce for ByteBool { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { Ok(Some( - ByteBoolData::new( + ByteBool::new( array.buffer().slice(range.clone()), array.validity().slice(range)?, ) diff --git a/encodings/datetime-parts/src/array.rs b/encodings/datetime-parts/src/array.rs index 54cbb6528e3..841586aa0b8 100644 --- a/encodings/datetime-parts/src/array.rs +++ b/encodings/datetime-parts/src/array.rs @@ -4,6 +4,7 @@ use std::fmt::Debug; use vortex_array::Array; +use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; @@ -22,7 +23,6 @@ use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityChild; @@ -74,28 +74,15 @@ impl DateTimePartsMetadata { impl VTable for DateTimeParts { type ArrayData = DateTimePartsData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; - fn vtable(_array: &Self::ArrayData) -> &Self { - &DateTimeParts - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &DateTimePartsData) -> usize { - array.days().len() - } - - fn dtype(array: &DateTimePartsData) -> &DType { - &array.dtype - } - - fn stats(array: &DateTimePartsData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + DateTimePartsData::validate(dtype, data.days(), data.seconds(), data.subseconds(), len) } fn array_hash( @@ -130,37 +117,25 @@ impl VTable for DateTimeParts { vortex_panic!("DateTimePartsArray buffer_name index {idx} out of bounds") } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(DateTimePartsMetadata { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some(ProstMetadata(DateTimePartsMetadata { days_ptype: PType::try_from(array.days().dtype())? as i32, seconds_ptype: PType::try_from(array.seconds().dtype())? as i32, subseconds_ptype: PType::try_from(array.subseconds().dtype())? as i32, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + }) + .serialize())) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata( - as DeserializeMetadata>::deserialize(bytes)?, - )) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = ProstMetadata::::deserialize(metadata)?; if children.len() != 3 { vortex_bail!( "Expected 3 children for datetime-parts encoding, found {}", @@ -170,17 +145,17 @@ impl VTable for DateTimeParts { let days = children.get( 0, - &DType::Primitive(metadata.0.get_days_ptype()?, dtype.nullability()), + &DType::Primitive(metadata.get_days_ptype()?, dtype.nullability()), len, )?; let seconds = children.get( 1, - &DType::Primitive(metadata.0.get_seconds_ptype()?, Nullability::NonNullable), + &DType::Primitive(metadata.get_seconds_ptype()?, Nullability::NonNullable), len, )?; let subseconds = children.get( 2, - &DType::Primitive(metadata.0.get_subseconds_ptype()?, Nullability::NonNullable), + &DType::Primitive(metadata.get_subseconds_ptype()?, Nullability::NonNullable), len, )?; @@ -241,9 +216,7 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["days", "seconds", "subseconds #[derive(Clone, Debug)] pub struct DateTimePartsData { - dtype: DType, pub(super) slots: Vec>, - stats_set: ArrayStats, } #[derive(Clone, Debug)] @@ -267,24 +240,30 @@ impl DateTimeParts { seconds: ArrayRef, subseconds: ArrayRef, ) -> VortexResult { - Array::try_from_data(DateTimePartsData::try_new( - dtype, days, seconds, subseconds, - )?) + let data = DateTimePartsData::try_new(dtype.clone(), days, seconds, subseconds)?; + let len = data.len(); + Array::try_from_parts(ArrayNew::new(DateTimeParts, dtype, len, data)) } /// Construct a [`DateTimePartsArray`] from a [`TemporalArray`]. pub fn try_from_temporal(temporal: TemporalArray) -> VortexResult { - Array::try_from_data(DateTimePartsData::try_from(temporal)?) + let dtype = temporal.dtype().clone(); + let data = DateTimePartsData::try_from(temporal)?; + let len = data.len(); + Array::try_from_parts(ArrayNew::new(DateTimeParts, dtype, len, data)) } } impl DateTimePartsData { - pub fn try_new( - dtype: DType, - days: ArrayRef, - seconds: ArrayRef, - subseconds: ArrayRef, - ) -> VortexResult { + pub fn validate( + dtype: &DType, + days: &ArrayRef, + seconds: &ArrayRef, + subseconds: &ArrayRef, + len: usize, + ) -> VortexResult<()> { + vortex_ensure!(days.len() == len, "expected len {len}, got {}", days.len()); + if !days.dtype().is_int() || (dtype.is_nullable() != days.dtype().is_nullable()) { vortex_bail!( "Expected integer with nullability {}, got {}", @@ -299,8 +278,7 @@ impl DateTimePartsData { vortex_bail!(MismatchedTypes: "non-nullable integer", subseconds.dtype()); } - let length = days.len(); - if length != seconds.len() || length != subseconds.len() { + if len != seconds.len() || len != subseconds.len() { vortex_bail!( "Mismatched lengths {} {} {}", days.len(), @@ -309,39 +287,19 @@ impl DateTimePartsData { ); } - Ok(Self { - dtype, - slots: vec![Some(days), Some(seconds), Some(subseconds)], - stats_set: Default::default(), - }) + Ok(()) } - pub(crate) unsafe fn new_unchecked( + pub fn try_new( dtype: DType, days: ArrayRef, seconds: ArrayRef, subseconds: ArrayRef, - ) -> Self { - Self { - dtype, + ) -> VortexResult { + Self::validate(&dtype, &days, &seconds, &subseconds, days.len())?; + Ok(Self { slots: vec![Some(days), Some(seconds), Some(subseconds)], - stats_set: Default::default(), - } - } - - pub fn into_parts(mut self) -> DateTimePartsArrayParts { - DateTimePartsArrayParts { - dtype: self.dtype, - days: self.slots[DAYS_SLOT] - .take() - .vortex_expect("DateTimePartsArray days slot"), - seconds: self.slots[SECONDS_SLOT] - .take() - .vortex_expect("DateTimePartsArray seconds slot"), - subseconds: self.slots[SUBSECONDS_SLOT] - .take() - .vortex_expect("DateTimePartsArray subseconds slot"), - } + }) } /// Returns the number of elements in the array. @@ -354,11 +312,6 @@ impl DateTimePartsData { self.days().len() == 0 } - /// Returns the logical data type of the array. - pub fn dtype(&self) -> &DType { - &self.dtype - } - pub fn days(&self) -> &ArrayRef { self.slots[DAYS_SLOT] .as_ref() diff --git a/encodings/datetime-parts/src/compute/cast.rs b/encodings/datetime-parts/src/compute/cast.rs index 9c0c405827d..cc60e253aaf 100644 --- a/encodings/datetime-parts/src/compute/cast.rs +++ b/encodings/datetime-parts/src/compute/cast.rs @@ -10,7 +10,6 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::DateTimeParts; -use crate::DateTimePartsData; impl CastReduce for DateTimeParts { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { if !array.dtype().eq_ignore_nullability(dtype) { @@ -18,7 +17,7 @@ impl CastReduce for DateTimeParts { }; Ok(Some( - DateTimePartsData::try_new( + DateTimeParts::try_new( dtype.clone(), array .days() diff --git a/encodings/datetime-parts/src/compute/filter.rs b/encodings/datetime-parts/src/compute/filter.rs index ba339063540..0320c5c1917 100644 --- a/encodings/datetime-parts/src/compute/filter.rs +++ b/encodings/datetime-parts/src/compute/filter.rs @@ -9,11 +9,10 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::DateTimeParts; -use crate::DateTimePartsData; impl FilterReduce for DateTimeParts { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { Ok(Some( - DateTimePartsData::try_new( + DateTimeParts::try_new( array.dtype().clone(), array.days().filter(mask.clone())?, array.seconds().filter(mask.clone())?, diff --git a/encodings/datetime-parts/src/compute/mask.rs b/encodings/datetime-parts/src/compute/mask.rs index 023831d251a..290707ef8ef 100644 --- a/encodings/datetime-parts/src/compute/mask.rs +++ b/encodings/datetime-parts/src/compute/mask.rs @@ -9,13 +9,12 @@ use vortex_array::scalar_fn::fns::mask::MaskReduce; use vortex_error::VortexResult; use crate::DateTimeParts; -use crate::DateTimePartsData; impl MaskReduce for DateTimeParts { fn mask(array: ArrayView<'_, Self>, mask: &ArrayRef) -> VortexResult> { let masked_days = array.days().clone().mask(mask.clone())?; Ok(Some( - DateTimePartsData::try_new( + DateTimeParts::try_new( array.dtype().as_nullable(), masked_days, array.seconds().clone(), diff --git a/encodings/datetime-parts/src/compute/rules.rs b/encodings/datetime-parts/src/compute/rules.rs index dedc87caa13..a50eb98f95c 100644 --- a/encodings/datetime-parts/src/compute/rules.rs +++ b/encodings/datetime-parts/src/compute/rules.rs @@ -26,7 +26,6 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::DateTimeParts; -use crate::DateTimePartsData; use crate::timestamp; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&DTPFilterPushDownRule), @@ -57,7 +56,7 @@ impl ArrayParentReduceRule for DTPFilterPushDownRule { return Ok(None); } - DateTimePartsData::try_new( + DateTimeParts::try_new( child.dtype().clone(), child.days().clone().filter(parent.filter_mask().clone())?, ConstantArray::new( diff --git a/encodings/datetime-parts/src/compute/slice.rs b/encodings/datetime-parts/src/compute/slice.rs index b63693980f7..5bad980d8dc 100644 --- a/encodings/datetime-parts/src/compute/slice.rs +++ b/encodings/datetime-parts/src/compute/slice.rs @@ -10,19 +10,18 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::DateTimeParts; -use crate::DateTimePartsData; impl SliceReduce for DateTimeParts { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { // SAFETY: slicing all components preserves values - Ok(Some(unsafe { - DateTimePartsData::new_unchecked( + Ok(Some( + DateTimeParts::try_new( array.dtype().clone(), array.days().slice(range.clone())?, array.seconds().slice(range.clone())?, array.subseconds().slice(range)?, - ) - .into_array() - })) + )? + .into_array(), + )) } } diff --git a/encodings/datetime-parts/src/compute/take.rs b/encodings/datetime-parts/src/compute/take.rs index b804752657b..436e7b2d38f 100644 --- a/encodings/datetime-parts/src/compute/take.rs +++ b/encodings/datetime-parts/src/compute/take.rs @@ -16,7 +16,6 @@ use vortex_error::VortexResult; use vortex_error::vortex_panic; use crate::DateTimeParts; -use crate::DateTimePartsData; fn take_datetime_parts( array: ArrayView, indices: &ArrayRef, @@ -38,10 +37,8 @@ fn take_datetime_parts( }; if !taken_seconds.dtype().is_nullable() && !taken_subseconds.dtype().is_nullable() { - return Ok( - DateTimePartsData::try_new(dtype, taken_days, taken_seconds, taken_subseconds)? - .into_array(), - ); + return Ok(DateTimeParts::try_new(dtype, taken_days, taken_seconds, taken_subseconds)? + .into_array()); } // DateTimePartsArray requires seconds and subseconds to be non-nullable. @@ -78,10 +75,7 @@ fn take_datetime_parts( .cast(array.subseconds().dtype())?; let taken_subseconds = taken_subseconds.fill_null(subseconds_fill)?; - Ok( - DateTimePartsData::try_new(dtype, taken_days, taken_seconds, taken_subseconds)? - .into_array(), - ) + Ok(DateTimeParts::try_new(dtype, taken_days, taken_seconds, taken_subseconds)?.into_array()) } impl TakeExecute for DateTimeParts { diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs index 5314dd8845d..d78c6e2a96d 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs @@ -7,6 +7,7 @@ use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::DecimalByteParts; @@ -19,7 +20,11 @@ impl CastReduce for DecimalByteParts { }; // Check if this is just a nullability change - if array.decimal_dtype() == target_decimal + if array + .dtype() + .as_decimal_opt() + .vortex_expect("must be a decimal dtype") + == target_decimal && array.dtype().nullability() != *target_nullability { // Cast the msp array to handle nullability change diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs index b1be5b1b54d..b0afad10cfa 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs @@ -38,7 +38,7 @@ impl CompareKernel for DecimalByteParts { return Ok(None); }; - let nullability = lhs.dtype.nullability() | rhs.dtype().nullability(); + let nullability = lhs.dtype().nullability() | rhs.dtype().nullability(); let scalar_type = lhs.msp().dtype().with_nullability(nullability); let rhs_decimal = rhs_const diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs index 638c167e9ed..49435845476 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs @@ -5,14 +5,21 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::arrays::filter::FilterReduce; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_mask::Mask; use crate::DecimalByteParts; impl FilterReduce for DecimalByteParts { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { - DecimalByteParts::try_new(array.msp().filter(mask.clone())?, *array.decimal_dtype()) - .map(|d| Some(d.into_array())) + DecimalByteParts::try_new( + array.msp().filter(mask.clone())?, + *array + .dtype() + .as_decimal_opt() + .vortex_expect("must be a decimal dtype"), + ) + .map(|d| Some(d.into_array())) } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs index f3d5068faa0..4c1b1a36c64 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs @@ -8,9 +8,9 @@ use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; use vortex_array::scalar_fn::EmptyOptions; use vortex_array::scalar_fn::fns::mask::Mask as MaskExpr; use vortex_array::scalar_fn::fns::mask::MaskReduce; +use vortex_error::VortexExpect; use vortex_error::VortexResult; -use super::DecimalBytePartsData; use crate::DecimalByteParts; impl MaskReduce for DecimalByteParts { @@ -21,7 +21,14 @@ impl MaskReduce for DecimalByteParts { [array.msp().clone(), mask.clone()], )?; Ok(Some( - DecimalBytePartsData::try_new(masked_msp, *array.decimal_dtype())?.into_array(), + DecimalByteParts::try_new( + masked_msp, + *array + .dtype() + .as_decimal_opt() + .vortex_expect("must be a decimal dtype"), + )? + .into_array(), )) } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs index 9aa64aa8d37..3c7f7ecfabf 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs @@ -6,9 +6,9 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::dict::TakeExecute; +use vortex_error::VortexExpect; use vortex_error::VortexResult; -use super::DecimalBytePartsData; use crate::DecimalByteParts; impl TakeExecute for DecimalByteParts { @@ -17,7 +17,13 @@ impl TakeExecute for DecimalByteParts { indices: &ArrayRef, _ctx: &mut ExecutionCtx, ) -> VortexResult> { - DecimalBytePartsData::try_new(array.msp().take(indices.clone())?, *array.decimal_dtype()) - .map(|a| Some(a.into_array())) + DecimalByteParts::try_new( + array.msp().take(indices.clone())?, + *array + .dtype() + .as_decimal_opt() + .vortex_expect("must be a decimal dtype"), + ) + .map(|a| Some(a.into_array())) } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs index 5fff2a47bf5..ffea1d4ce41 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::Array; +use vortex_array::ArrayNew; use vortex_array::ArrayView; pub(crate) mod compute; mod rules; @@ -29,7 +30,6 @@ use vortex_array::scalar::DecimalValue; use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; use vortex_array::vtable::VTable; @@ -58,28 +58,18 @@ pub struct DecimalBytesPartsMetadata { impl VTable for DecimalByteParts { type ArrayData = DecimalBytePartsData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; - fn vtable(_array: &Self::ArrayData) -> &Self { - &DecimalByteParts - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &DecimalBytePartsData) -> usize { - array.msp().len() - } - - fn dtype(array: &DecimalBytePartsData) -> &DType { - &array.dtype - } - - fn stats(array: &DecimalBytePartsData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + let Some(decimal_dtype) = dtype.as_decimal_opt() else { + vortex_bail!("expected decimal dtype, got {}", dtype) + }; + DecimalBytePartsData::validate(data.msp(), *decimal_dtype, dtype, len) } fn array_hash( @@ -110,34 +100,24 @@ impl VTable for DecimalByteParts { vortex_panic!("DecimalBytePartsArray buffer_name index {idx} out of bounds") } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(DecimalBytesPartsMetadata { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some(ProstMetadata(DecimalBytesPartsMetadata { zeroth_child_ptype: PType::try_from(array.msp().dtype())? as i32, lower_part_count: 0, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + }) + .serialize())) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata(DecimalBytesPartsMetadata::decode(bytes)?)) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = DecimalBytesPartsMetadata::decode(metadata)?; let Some(decimal_dtype) = dtype.as_decimal_opt() else { vortex_bail!("decoding decimal but given non decimal dtype {}", dtype) }; @@ -213,8 +193,6 @@ pub struct DecimalBytePartsData { // other than the empty Vec. // Must update `DecimalBytePartsArrayParts` too. _lower_parts: Vec, - dtype: DType, - stats_set: ArrayStats, } pub struct DecimalBytePartsArrayParts { @@ -223,38 +201,29 @@ pub struct DecimalBytePartsArrayParts { } impl DecimalBytePartsData { - pub fn try_new(msp: ArrayRef, decimal_dtype: DecimalDType) -> VortexResult { + pub fn validate( + msp: &ArrayRef, + decimal_dtype: DecimalDType, + dtype: &DType, + len: usize, + ) -> VortexResult<()> { if !msp.dtype().is_signed_int() { vortex_bail!("decimal bytes parts, first part must be a signed array") } - let nullable = msp.dtype().nullability(); - Ok(Self { - slots: vec![Some(msp)], - _lower_parts: Vec::new(), - dtype: DType::Decimal(decimal_dtype, nullable), - stats_set: Default::default(), - }) + let expected_dtype = DType::Decimal(decimal_dtype, msp.dtype().nullability()); + vortex_ensure!(dtype == &expected_dtype, "expected dtype {expected_dtype}, got {dtype}"); + vortex_ensure!(msp.len() == len, "expected len {len}, got {}", msp.len()); + Ok(()) } - pub(crate) unsafe fn new_unchecked(msp: ArrayRef, decimal_dtype: DecimalDType) -> Self { - let nullable = msp.dtype().nullability(); - Self { + pub fn try_new(msp: ArrayRef, decimal_dtype: DecimalDType) -> VortexResult { + let dtype = DType::Decimal(decimal_dtype, msp.dtype().nullability()); + Self::validate(&msp, decimal_dtype, &dtype, msp.len())?; + Ok(Self { slots: vec![Some(msp)], _lower_parts: Vec::new(), - dtype: DType::Decimal(decimal_dtype, nullable), - stats_set: Default::default(), - } - } - - /// If `_lower_parts` is supported check all calls use this correctly. - pub fn into_parts(mut self) -> DecimalBytePartsArrayParts { - DecimalBytePartsArrayParts { - msp: self.slots[MSP_SLOT] - .take() - .vortex_expect("DecimalBytePartsArray msp slot"), - dtype: self.dtype, - } + }) } /// Returns the number of elements in the array. @@ -267,17 +236,6 @@ impl DecimalBytePartsData { self.msp().len() == 0 } - /// Returns the logical data type of the array. - pub fn dtype(&self) -> &DType { - &self.dtype - } - - pub fn decimal_dtype(&self) -> &DecimalDType { - self.dtype - .as_decimal_opt() - .vortex_expect("must be a decimal dtype") - } - pub(crate) fn msp(&self) -> &ArrayRef { self.slots[MSP_SLOT] .as_ref() @@ -296,7 +254,10 @@ impl DecimalByteParts { msp: ArrayRef, decimal_dtype: DecimalDType, ) -> VortexResult { - Array::try_from_data(DecimalBytePartsData::try_new(msp, decimal_dtype)?) + let data = DecimalBytePartsData::try_new(msp, decimal_dtype)?; + let dtype = DType::Decimal(decimal_dtype, data.msp().dtype().nullability()); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(DecimalByteParts, dtype, len, data)) } } @@ -316,7 +277,10 @@ fn to_canonical_decimal( unsafe { DecimalArray::new_unchecked( prim.to_buffer::

(), - *array.decimal_dtype(), + *array + .dtype() + .as_decimal_opt() + .vortex_expect("must be a decimal dtype"), prim.validity(), ) } @@ -338,7 +302,7 @@ impl OperationsVTable for DecimalByteParts { // TODO(joe): extend this to support multiple parts. let value = primitive_scalar.as_::().vortex_expect("non-null"); Scalar::try_new( - array.dtype.clone(), + array.dtype().clone(), Some(ScalarValue::Decimal(DecimalValue::I64(value))), ) } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs index f9b4422f7a6..0eeaeed5716 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs @@ -11,9 +11,9 @@ use vortex_array::optimizer::rules::ArrayParentReduceRule; use vortex_array::optimizer::rules::ParentRuleSet; use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; use vortex_array::scalar_fn::fns::mask::MaskReduceAdaptor; +use vortex_error::VortexExpect; use vortex_error::VortexResult; -use super::DecimalBytePartsData; use crate::DecimalByteParts; pub(super) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ @@ -43,8 +43,14 @@ impl ArrayParentReduceRule for DecimalBytePartsFilterPushDownR } let new_msp = child.msp().filter(parent.filter_mask().clone())?; - let new_child = - DecimalBytePartsData::try_new(new_msp, *child.decimal_dtype())?.into_array(); + let new_child = DecimalByteParts::try_new( + new_msp, + *child + .dtype() + .as_decimal_opt() + .vortex_expect("must be a decimal dtype"), + )? + .into_array(); Ok(Some(new_child)) } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/slice.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/slice.rs index 051d80ff016..89275570a53 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/slice.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/slice.rs @@ -7,17 +7,22 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::arrays::slice::SliceReduce; +use vortex_error::VortexExpect; use vortex_error::VortexResult; -use super::DecimalBytePartsData; use crate::DecimalByteParts; impl SliceReduce for DecimalByteParts { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - // SAFETY: slicing encoded MSP does not change the encoded values - Ok(Some(unsafe { - DecimalBytePartsData::new_unchecked(array.msp().slice(range)?, *array.decimal_dtype()) - .into_array() - })) + Ok(Some( + DecimalByteParts::try_new( + array.msp().slice(range)?, + *array + .dtype() + .as_decimal_opt() + .vortex_expect("must be a decimal dtype"), + )? + .into_array(), + )) } } diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index 1365d8ad087..184d061e8ed 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -16,6 +16,7 @@ use pco::wrapped::FileCompressor; use pco::wrapped::FileDecompressor; use prost::Message; use vortex_array::Array; +use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; @@ -26,7 +27,6 @@ use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; use vortex_array::Precision; -use vortex_array::ProstMetadata; use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::Primitive; @@ -37,7 +37,6 @@ use vortex_array::dtype::PType; use vortex_array::dtype::half; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; @@ -84,28 +83,15 @@ vtable!(Pco, Pco, PcoData); impl VTable for Pco { type ArrayData = PcoData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromValiditySliceHelper; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Pco - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &PcoData) -> usize { - array.slice_stop - array.slice_start - } - - fn dtype(array: &PcoData) -> &DType { - &array.dtype - } - - fn stats(array: &PcoData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &PcoData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate(dtype, len) } fn array_hash(array: &PcoData, state: &mut H, precision: Precision) { @@ -168,31 +154,20 @@ impl VTable for Pco { } } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(array.metadata.clone())) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.0.encode_to_vec())) + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some(array.metadata.clone().encode_to_vec())) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata(PcoMetadata::decode(bytes)?)) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = PcoMetadata::decode(metadata)?; let validity = if children.is_empty() { Validity::from(dtype.nullability()) } else if children.len() == 1 { @@ -202,18 +177,17 @@ impl VTable for Pco { vortex_bail!("PcoArray expected 0 or 1 child, got {}", children.len()); }; - vortex_ensure!(buffers.len() >= metadata.0.chunks.len()); - let chunk_metas = buffers[..metadata.0.chunks.len()] + vortex_ensure!(buffers.len() >= metadata.chunks.len()); + let chunk_metas = buffers[..metadata.chunks.len()] .iter() .map(|b| b.clone().try_to_host_sync()) .collect::>>()?; - let pages = buffers[metadata.0.chunks.len()..] + let pages = buffers[metadata.chunks.len()..] .iter() .map(|b| b.clone().try_to_host_sync()) .collect::>>()?; let expected_n_pages = metadata - .0 .chunks .iter() .map(|info| info.pages.len()) @@ -223,8 +197,8 @@ impl VTable for Pco { Ok(PcoData::new( chunk_metas, pages, - dtype.clone(), - metadata.0.clone(), + dtype.as_ptype(), + metadata, len, validity, )) @@ -247,7 +221,7 @@ impl VTable for Pco { ); array.unsliced_validity = match &slots[VALIDITY_SLOT] { Some(arr) => Validity::Array(arr.clone()), - None => Validity::from(array.dtype.nullability()), + None => Validity::from(array.unsliced_validity.nullability()), }; array.slots = slots; Ok(()) @@ -267,7 +241,10 @@ impl VTable for Pco { } pub(crate) fn number_type_from_dtype(dtype: &DType) -> NumberType { - let ptype = dtype.as_ptype(); + number_type_from_ptype(dtype.as_ptype()) +} + +pub(crate) fn number_type_from_ptype(ptype: PType) -> NumberType { match ptype { PType::F16 => NumberType::F16, PType::F32 => NumberType::F32, @@ -302,13 +279,20 @@ pub struct Pco; impl Pco { pub const ID: ArrayId = ArrayId::new_ref("vortex.pco"); + pub(crate) fn try_new(dtype: DType, data: PcoData) -> VortexResult { + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Pco, dtype, len, data)) + } + /// Compress a primitive array using pcodec. pub fn from_primitive( parray: &PrimitiveArray, level: usize, values_per_page: usize, ) -> VortexResult { - Array::try_from_data(PcoData::from_primitive(parray, level, values_per_page)?) + let dtype = parray.dtype().clone(); + let data = PcoData::from_primitive(parray, level, values_per_page)?; + Self::try_new(dtype, data) } } @@ -322,20 +306,66 @@ pub struct PcoData { pub(crate) chunk_metas: Vec, pub(crate) pages: Vec, pub(crate) metadata: PcoMetadata, - dtype: DType, + ptype: PType, pub(crate) unsliced_validity: Validity, unsliced_n_rows: usize, pub(super) slots: Vec>, - stats_set: ArrayStats, slice_start: usize, slice_stop: usize, } impl PcoData { + pub fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { + let _ = number_type_from_ptype(self.ptype); + vortex_ensure!( + dtype.as_ptype() == self.ptype, + "expected ptype {}, got {}", + self.ptype, + dtype.as_ptype() + ); + vortex_ensure!( + dtype.nullability() == self.unsliced_validity.nullability(), + "expected nullability {}, got {}", + self.unsliced_validity.nullability(), + dtype.nullability() + ); + vortex_ensure!( + self.slice_start <= self.slice_stop && self.slice_stop <= self.unsliced_n_rows, + "invalid slice range {}..{} for {} rows", + self.slice_start, + self.slice_stop, + self.unsliced_n_rows + ); + vortex_ensure!( + self.slice_stop - self.slice_start == len, + "expected len {len}, got {}", + self.slice_stop - self.slice_start + ); + if let Some(validity_len) = self.unsliced_validity.maybe_len() { + vortex_ensure!( + validity_len == self.unsliced_n_rows, + "expected validity len {}, got {}", + self.unsliced_n_rows, + validity_len + ); + } + vortex_ensure!( + self.chunk_metas.len() == self.metadata.chunks.len(), + "expected {} chunk metas, got {}", + self.metadata.chunks.len(), + self.chunk_metas.len() + ); + vortex_ensure!( + self.pages.len() == self.metadata.chunks.iter().map(|chunk| chunk.pages.len()).sum(), + "page count does not match metadata" + ); + Ok(()) + } + pub fn new( chunk_metas: Vec, pages: Vec, - dtype: DType, + ptype: PType, metadata: PcoMetadata, len: usize, validity: Validity, @@ -346,11 +376,10 @@ impl PcoData { chunk_metas, pages, metadata, - dtype, + ptype, unsliced_validity: validity, unsliced_n_rows: len, slots: vec![validity_slot], - stats_set: Default::default(), slice_start: 0, slice_stop: len, } @@ -430,7 +459,7 @@ impl PcoData { Ok(PcoData::new( chunk_meta_buffers, page_buffers, - parray.dtype().clone(), + parray.dtype().as_ptype(), metadata, parray.len(), parray.validity(), @@ -450,7 +479,7 @@ impl PcoData { pub fn decompress(&self, ctx: &mut ExecutionCtx) -> VortexResult { // To start, we figure out which chunks and pages we need to decompress, and with // what value offset into the first such page. - let number_type = number_type_from_dtype(&self.dtype); + let number_type = number_type_from_ptype(self.ptype); let values_byte_buffer = match_number_enum!( number_type, NumberType => { @@ -460,7 +489,7 @@ impl PcoData { Ok(PrimitiveArray::from_values_byte_buffer( values_byte_buffer, - self.dtype.as_ptype(), + self.ptype, self.unsliced_validity .slice(self.slice_start..self.slice_stop)?, self.slice_stop - self.slice_start, @@ -547,7 +576,6 @@ impl PcoData { PcoData { slice_start: self.slice_start + start, slice_stop: self.slice_start + stop, - stats_set: Default::default(), ..self.clone() } } @@ -562,11 +590,6 @@ impl PcoData { self.slice_stop == self.slice_start } - /// Returns the logical data type of the array. - pub fn dtype(&self) -> &DType { - &self.dtype - } - pub(crate) fn slice_start(&self) -> usize { self.slice_start } diff --git a/encodings/pco/src/compute/cast.rs b/encodings/pco/src/compute/cast.rs index 378f07254ac..3b566eb390e 100644 --- a/encodings/pco/src/compute/cast.rs +++ b/encodings/pco/src/compute/cast.rs @@ -29,16 +29,18 @@ impl CastReduce for Pco { .clone() .cast_nullability(dtype.nullability(), array.len())?; + let data = PcoData::new( + array.chunk_metas.clone(), + array.pages.clone(), + dtype.as_ptype(), + array.metadata.clone(), + array.unsliced_n_rows(), + new_validity, + ) + ._slice(array.slice_start(), array.slice_stop()); + return Ok(Some( - PcoData::new( - array.chunk_metas.clone(), - array.pages.clone(), - dtype.clone(), - array.metadata.clone(), - array.unsliced_n_rows(), - new_validity, - ) - ._slice(array.slice_start(), array.slice_stop()) + Pco::try_new(dtype.clone(), data)? .into_array(), )); } diff --git a/encodings/pco/src/slice.rs b/encodings/pco/src/slice.rs index bdaa1006294..a53a1e39898 100644 --- a/encodings/pco/src/slice.rs +++ b/encodings/pco/src/slice.rs @@ -13,6 +13,8 @@ use crate::Pco; impl SliceReduce for Pco { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - Ok(Some(array._slice(range.start, range.end).into_array())) + Ok(Some( + Pco::try_new(array.dtype().clone(), array._slice(range.start, range.end))?.into_array(), + )) } } diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index 081b48ab678..8d52c21858e 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -5,6 +5,7 @@ use std::fmt::Debug; use std::hash::Hash; use vortex_array::Array; +use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; @@ -27,7 +28,6 @@ use vortex_array::scalar::PValue; use vortex_array::search_sorted::SearchSorted; use vortex_array::search_sorted::SearchSortedSide; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; @@ -61,28 +61,22 @@ pub struct RunEndMetadata { impl VTable for RunEnd { type ArrayData = RunEndData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &RunEnd - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &RunEndData) -> usize { - array.length - } - - fn dtype(array: &RunEndData) -> &DType { - array.values().dtype() - } - - fn stats(array: &RunEndData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + RunEndData::validate(data.ends(), data.values(), data.offset, len)?; + vortex_ensure!( + data.values().dtype() == dtype, + "expected dtype {}, got {}", + dtype, + data.values().dtype() + ); + Ok(()) } fn array_hash(array: &RunEndData, state: &mut H, precision: Precision) { @@ -109,37 +103,26 @@ impl VTable for RunEnd { vortex_panic!("RunEndArray buffer_name index {idx} out of bounds") } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(RunEndMetadata { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some(ProstMetadata(RunEndMetadata { ends_ptype: PType::try_from(array.ends().dtype()).vortex_expect("Must be a valid PType") as i32, num_runs: array.ends().len() as u64, offset: array.offset() as u64, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + }) + .serialize())) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - let inner = as DeserializeMetadata>::deserialize(bytes)?; - Ok(ProstMetadata(inner)) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = ProstMetadata::::deserialize(metadata)?; let ends_dtype = DType::Primitive(metadata.ends_ptype(), Nullability::NonNullable); let runs = usize::try_from(metadata.num_runs).vortex_expect("Must be a valid usize"); let ends = children.get(0, &ends_dtype, runs)?; @@ -206,8 +189,6 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["ends", "values"]; pub struct RunEndData { pub(super) slots: Vec>, offset: usize, - length: usize, - stats_set: ArrayStats, } pub struct RunEndArrayParts { @@ -231,13 +212,18 @@ impl RunEnd { offset: usize, length: usize, ) -> RunEndArray { - Array::try_from_data(unsafe { RunEndData::new_unchecked(ends, values, offset, length) }) + let data = unsafe { RunEndData::new_unchecked(ends, values, offset, length) }; + let dtype = data.dtype().clone(); + Array::try_from_parts(ArrayNew::new(RunEnd, dtype, length, data)) .vortex_expect("RunEndData is always valid") } /// Build a new [`RunEndArray`] from ends and values. pub fn try_new(ends: ArrayRef, values: ArrayRef) -> VortexResult { - Array::try_from_data(RunEndData::try_new(ends, values)?) + let len = RunEndData::logical_len_from_ends(&ends)?; + let data = RunEndData::try_new_offset_length(ends, values, 0, len)?; + let dtype = data.dtype().clone(); + Array::try_from_parts(ArrayNew::new(RunEnd, dtype, len, data)) } /// Build a new [`RunEndArray`] from ends, values, offset, and length. @@ -247,24 +233,34 @@ impl RunEnd { offset: usize, length: usize, ) -> VortexResult { - Array::try_from_data(RunEndData::try_new_offset_length( - ends, values, offset, length, - )?) + let data = RunEndData::try_new_offset_length(ends, values, offset, length)?; + let dtype = data.dtype().clone(); + Array::try_from_parts(ArrayNew::new(RunEnd, dtype, length, data)) } /// Build a new [`RunEndArray`] from ends and values (panics on invalid input). pub fn new(ends: ArrayRef, values: ArrayRef) -> RunEndArray { - Array::try_from_data(RunEndData::new(ends, values)) - .vortex_expect("RunEndData is always valid") + Self::try_new(ends, values).vortex_expect("RunEndData is always valid") } /// Run the array through run-end encoding. pub fn encode(array: ArrayRef) -> VortexResult { - Array::try_from_data(RunEndData::encode(array)?) + let len = array.len(); + let data = RunEndData::encode(array)?; + let dtype = data.dtype().clone(); + Array::try_from_parts(ArrayNew::new(RunEnd, dtype, len, data)) } } impl RunEndData { + fn logical_len_from_ends(ends: &ArrayRef) -> VortexResult { + if ends.is_empty() { + Ok(0) + } else { + usize::try_from(&ends.scalar_at(ends.len() - 1)?) + } + } + fn validate( ends: &ArrayRef, values: &ArrayRef, @@ -376,12 +372,7 @@ impl RunEndData { /// /// The `ends` must be non-nullable unsigned integers. pub fn try_new(ends: ArrayRef, values: ArrayRef) -> VortexResult { - let length: usize = if ends.is_empty() { - 0 - } else { - usize::try_from(&ends.scalar_at(ends.len() - 1)?)? - }; - + let length = Self::logical_len_from_ends(&ends)?; Self::try_new_offset_length(ends, values, 0, length) } @@ -399,8 +390,6 @@ impl RunEndData { Ok(Self { slots: vec![Some(ends), Some(values)], offset, - length, - stats_set: Default::default(), }) } @@ -416,13 +405,11 @@ impl RunEndData { ends: ArrayRef, values: ArrayRef, offset: usize, - length: usize, + _length: usize, ) -> Self { Self { slots: vec![Some(ends), Some(values)], offset, - length, - stats_set: Default::default(), } } @@ -456,18 +443,6 @@ impl RunEndData { } } - /// Returns the length of the array. - #[inline] - pub fn len(&self) -> usize { - self.length - } - - /// Returns whether the array is empty. - #[inline] - pub fn is_empty(&self) -> bool { - self.length == 0 - } - /// Returns the logical data type of the array. #[inline] pub fn dtype(&self) -> &DType { @@ -524,7 +499,7 @@ impl ValidityVTable for RunEnd { Validity::NonNullable | Validity::AllValid => Validity::AllValid, Validity::AllInvalid => Validity::AllInvalid, Validity::Array(values_validity) => Validity::Array(unsafe { - RunEndData::new_unchecked( + RunEnd::new_unchecked( array.ends().clone(), values_validity, array.offset(), diff --git a/encodings/runend/src/compute/cast.rs b/encodings/runend/src/compute/cast.rs index a5fec75d802..d9ecb3a0baf 100644 --- a/encodings/runend/src/compute/cast.rs +++ b/encodings/runend/src/compute/cast.rs @@ -10,7 +10,6 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::RunEnd; -use crate::RunEndData; impl CastReduce for RunEnd { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // Cast the values array to the target type @@ -19,7 +18,7 @@ impl CastReduce for RunEnd { // SAFETY: casting does not affect the ends being valid unsafe { Ok(Some( - RunEndData::new_unchecked( + RunEnd::new_unchecked( array.ends().clone(), casted_values, array.offset(), diff --git a/encodings/runend/src/compute/fill_null.rs b/encodings/runend/src/compute/fill_null.rs index bb138e406d6..abe0ef44f09 100644 --- a/encodings/runend/src/compute/fill_null.rs +++ b/encodings/runend/src/compute/fill_null.rs @@ -10,7 +10,6 @@ use vortex_array::scalar_fn::fns::fill_null::FillNullReduce; use vortex_error::VortexResult; use crate::RunEnd; -use crate::RunEndData; impl FillNullReduce for RunEnd { fn fill_null( @@ -21,7 +20,7 @@ impl FillNullReduce for RunEnd { // SAFETY: modifying values only, does not affect ends Ok(Some( unsafe { - RunEndData::new_unchecked( + RunEnd::new_unchecked( array.ends().clone(), new_values, array.offset(), diff --git a/encodings/runend/src/compute/filter.rs b/encodings/runend/src/compute/filter.rs index 92c3ef40b8e..e0ef5382d40 100644 --- a/encodings/runend/src/compute/filter.rs +++ b/encodings/runend/src/compute/filter.rs @@ -21,7 +21,6 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::RunEnd; -use crate::RunEndData; use crate::compute::take::take_indices_unchecked; const FILTER_TAKE_THRESHOLD: f64 = 0.1; @@ -59,7 +58,7 @@ impl FilterKernel for RunEnd { // SAFETY: guaranteed by implementation of filter_run_end_primitive unsafe { Ok(Some( - RunEndData::new_unchecked( + RunEnd::new_unchecked( run_ends.into_array(), values, 0, diff --git a/encodings/runend/src/kernel.rs b/encodings/runend/src/kernel.rs index 2ca1586bb51..97f17c17857 100644 --- a/encodings/runend/src/kernel.rs +++ b/encodings/runend/src/kernel.rs @@ -17,7 +17,6 @@ use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor; use vortex_error::VortexResult; use crate::RunEnd; -use crate::RunEndData; use crate::compute::take_from::RunEndTakeFrom; pub(super) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ @@ -45,11 +44,11 @@ impl ExecuteParentKernel for RunEndSliceKernel { _child_idx: usize, _ctx: &mut ExecutionCtx, ) -> VortexResult> { - slice(&array, parent.slice_range().clone()).map(Some) + slice(array, parent.slice_range().clone()).map(Some) } } -fn slice(array: &RunEndData, range: Range) -> VortexResult { +fn slice(array: ArrayView<'_, RunEnd>, range: Range) -> VortexResult { let new_length = range.len(); let slice_begin = array.find_physical_index(range.start)?; @@ -63,7 +62,7 @@ fn slice(array: &RunEndData, range: Range) -> VortexResult { // SAFETY: we maintain the ends invariant in our slice implementation Ok(unsafe { - RunEndData::new_unchecked( + RunEnd::new_unchecked( array.ends().slice(slice_begin..slice_end)?, array.values().slice(slice_begin..slice_end)?, range.start + array.offset(), diff --git a/encodings/runend/src/rules.rs b/encodings/runend/src/rules.rs index 4ad3db969c3..2c8a6607156 100644 --- a/encodings/runend/src/rules.rs +++ b/encodings/runend/src/rules.rs @@ -17,7 +17,6 @@ use vortex_array::scalar_fn::fns::fill_null::FillNullReduceAdaptor; use vortex_error::VortexResult; use crate::RunEnd; -use crate::RunEndData; pub(super) const RULES: ParentRuleSet = ParentRuleSet::new(&[ // CastReduceAdaptor must come before RunEndScalarFnRule so that cast operations are executed @@ -81,7 +80,7 @@ impl ArrayParentReduceRule for RunEndScalarFnRule { Ok(Some( unsafe { - RunEndData::new_unchecked( + RunEnd::new_unchecked( run_end.ends().clone(), new_values, run_end.offset(), diff --git a/encodings/sequence/src/array.rs b/encodings/sequence/src/array.rs index 4b38b5f9124..7d3dab3291f 100644 --- a/encodings/sequence/src/array.rs +++ b/encodings/sequence/src/array.rs @@ -5,6 +5,7 @@ use std::hash::Hash; use num_traits::cast::FromPrimitive; use vortex_array::Array; +use vortex_array::ArrayNew; use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::ArrayView; @@ -50,12 +51,6 @@ use crate::rules::RULES; vtable!(Sequence, Sequence, SequenceData); -#[derive(Debug, Clone, Copy)] -pub struct SequenceMetadata { - base: PValue, - multiplier: PValue, -} - #[derive(Clone, prost::Message)] pub struct ProstSequenceMetadata { #[prost(message, tag = "1")] @@ -68,9 +63,7 @@ pub struct ProstSequenceMetadata { pub struct SequenceArrayParts { pub base: PValue, pub multiplier: PValue, - pub len: usize, pub ptype: PType, - pub nullability: Nullability, } pub(super) const SLOT_NAMES: [&str; 0] = []; @@ -80,10 +73,7 @@ pub(super) const SLOT_NAMES: [&str; 0] = []; pub struct SequenceData { base: PValue, multiplier: PValue, - dtype: DType, - pub(crate) len: usize, pub(super) slots: Vec>, - stats_set: ArrayStats, } impl SequenceData { @@ -110,92 +100,60 @@ impl SequenceData { nullability: Nullability, length: usize, ) -> VortexResult { + let dtype = DType::Primitive(ptype, nullability); + Self::validate(base, multiplier, &dtype, length)?; + let (base, multiplier) = Self::normalize(base, multiplier, ptype)?; + + Ok(unsafe { Self::new_unchecked(base, multiplier) }) + } + + pub fn validate( + base: PValue, + multiplier: PValue, + dtype: &DType, + length: usize, + ) -> VortexResult<()> { + let DType::Primitive(ptype, _) = dtype else { + vortex_bail!("only primitive dtypes are supported in SequenceArray currently"); + }; + if !ptype.is_int() { vortex_bail!("only integer ptype are supported in SequenceArray currently") } - Self::try_last(base, multiplier, ptype, length).map_err(|e| { + vortex_ensure!(length > 0, "SequenceArray length must be greater than zero"); + Self::try_last(base, multiplier, *ptype, length).map_err(|e| { e.with_context(format!( "final value not expressible, base = {base:?}, multiplier = {multiplier:?}, len = {length} ", )) })?; - // SAFETY: we just validated that `ptype` is an integer and that the final - // element is representable via `try_last`. - Ok(unsafe { Self::new_unchecked(base, multiplier, ptype, nullability, length) }) + Ok(()) } - /// Constructs a [`SequenceArray`] without validating that the `ptype` is an integer - /// type or that the final element is representable. + fn normalize(base: PValue, multiplier: PValue, ptype: PType) -> VortexResult<(PValue, PValue)> { + match_each_integer_ptype!(ptype, |P| { + Ok((PValue::from(base.cast::

()?), PValue::from(multiplier.cast::

()?))) + }) + } + + /// Constructs a [`SequenceArray`] payload without validation. /// /// # Safety /// /// The caller must ensure that: - /// - `ptype` is an integer type (i.e., `ptype.is_int()` returns `true`). - /// - `base + (length - 1) * multiplier` does not overflow the range of `ptype`. - /// - /// Violating the first invariant will cause a panic. Violating the second will - /// cause silent wraparound when materializing elements, producing incorrect values. - pub(crate) unsafe fn new_unchecked( - base: PValue, - multiplier: PValue, - ptype: PType, - nullability: Nullability, - length: usize, - ) -> Self { - let dtype = DType::Primitive(ptype, nullability); - - // A sequence A[i] = base + i * multiplier is sorted iff multiplier >= 0, - // and strictly sorted iff multiplier > 0. - - let (is_sorted, is_strict_sorted) = match_each_pvalue!( - multiplier, - uint: |v| { (true, v> 0) }, - int: |v| { (v >= 0, v > 0) }, - float: |_v| { unreachable!("float multiplier not supported") } - ); - - // SAFETY: we don't have duplicate stats - let stats_set = unsafe { - StatsSet::new_unchecked(vec![ - (Stat::IsSorted, StatPrecision::Exact(is_sorted.into())), - ( - Stat::IsStrictSorted, - StatPrecision::Exact(is_strict_sorted.into()), - ), - ]) - }; - + /// - `base` and `multiplier` are both normalized to the same integer `ptype`. + /// - they are logically compatible with the outer dtype and len. + pub(crate) unsafe fn new_unchecked(base: PValue, multiplier: PValue) -> Self { Self { base, multiplier, - dtype, - len: length, slots: vec![], - stats_set: ArrayStats::from(stats_set), } } - /// Returns the length of the array. - #[inline] - pub fn len(&self) -> usize { - self.len - } - - /// Returns whether the array is empty. - #[inline] - pub fn is_empty(&self) -> bool { - self.len == 0 - } - - /// Returns the logical data type of the array. - #[inline] - pub fn dtype(&self) -> &DType { - &self.dtype - } - pub fn ptype(&self) -> PType { - self.dtype.as_ptype() + self.base.ptype() } pub fn base(&self) -> PValue { @@ -227,8 +185,6 @@ impl SequenceData { } pub(crate) fn index_value(&self, idx: usize) -> PValue { - assert!(idx < self.len, "index_value({idx}): index out of bounds"); - match_each_native_ptype!(self.ptype(), |P| { let base = self.base.cast::

().vortex_expect("must be able to cast"); let multiplier = self @@ -241,19 +197,11 @@ impl SequenceData { }) } - /// Returns the validated final value of a sequence array - pub fn last(&self) -> PValue { - Self::try_last(self.base, self.multiplier, self.ptype(), self.len) - .vortex_expect("validated array") - } - pub fn into_parts(self) -> SequenceArrayParts { SequenceArrayParts { base: self.base, multiplier: self.multiplier, - len: self.len, - ptype: self.dtype.as_ptype(), - nullability: self.dtype.nullability(), + ptype: self.base.ptype(), } } } @@ -261,28 +209,15 @@ impl SequenceData { impl VTable for Sequence { type ArrayData = SequenceData; - type Metadata = SequenceMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Sequence - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &SequenceData) -> usize { - array.len - } - - fn dtype(array: &SequenceData) -> &DType { - &array.dtype - } - - fn stats(array: &SequenceData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + SequenceData::validate(data.base, data.multiplier, dtype, len) } fn array_hash( @@ -310,31 +245,28 @@ impl VTable for Sequence { vortex_panic!("SequenceArray buffer_name index {idx} out of bounds") } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(SequenceMetadata { - base: array.base(), - multiplier: array.multiplier(), - }) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { let prost = ProstMetadata(ProstSequenceMetadata { - base: Some((&metadata.base).into()), - multiplier: Some((&metadata.multiplier).into()), + base: Some((&array.base()).into()), + multiplier: Some((&array.multiplier()).into()), }); Ok(Some(prost.serialize())) } fn deserialize( - bytes: &[u8], + &self, dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], + len: usize, + metadata: &[u8], + buffers: &[BufferHandle], + children: &dyn ArrayChildren, session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult { + vortex_ensure!(buffers.is_empty(), "SequenceArray expects 0 buffers, got {}", buffers.len()); + vortex_ensure!(children.is_empty(), "SequenceArray expects 0 children, got {}", children.len()); let prost = - as DeserializeMetadata>::deserialize(bytes)?; + as DeserializeMetadata>::deserialize(metadata)?; let ptype = dtype.as_ptype(); @@ -363,20 +295,10 @@ impl VTable for Sequence { .pvalue() .vortex_expect("sequence array multiplier should be a non-nullable primitive"); - Ok(SequenceMetadata { base, multiplier }) - } - - fn build( - dtype: &DType, - len: usize, - metadata: &Self::Metadata, - _buffers: &[BufferHandle], - _children: &dyn ArrayChildren, - ) -> VortexResult { SequenceData::try_new( - metadata.base, - metadata.multiplier, - dtype.as_ptype(), + base, + multiplier, + ptype, dtype.nullability(), len, ) @@ -447,6 +369,50 @@ pub struct Sequence; impl Sequence { pub const ID: ArrayId = ArrayId::new_ref("vortex.sequence"); + fn stats(multiplier: PValue) -> ArrayStats { + // A sequence A[i] = base + i * multiplier is sorted iff multiplier >= 0, + // and strictly sorted iff multiplier > 0. + let (is_sorted, is_strict_sorted) = match_each_pvalue!( + multiplier, + uint: |v| { (true, v > 0) }, + int: |v| { (v >= 0, v > 0) }, + float: |_v| { unreachable!("float multiplier not supported") } + ); + + // SAFETY: we don't have duplicate stats. + let stats_set = unsafe { + StatsSet::new_unchecked(vec![ + (Stat::IsSorted, StatPrecision::Exact(is_sorted.into())), + ( + Stat::IsStrictSorted, + StatPrecision::Exact(is_strict_sorted.into()), + ), + ]) + }; + ArrayStats::from(stats_set) + } + + /// Construct a new [`SequenceArray`] from pre-validated parts. + /// + /// # Safety + /// + /// Caller must ensure the sequence is logically compatible with the provided dtype and len. + pub(crate) unsafe fn new_unchecked( + base: PValue, + multiplier: PValue, + ptype: PType, + nullability: Nullability, + length: usize, + ) -> SequenceArray { + let dtype = DType::Primitive(ptype, nullability); + let (base, multiplier) = SequenceData::normalize(base, multiplier, ptype) + .vortex_expect("SequenceArray parts must be normalized to the target ptype"); + let stats = Self::stats(multiplier); + let data = unsafe { SequenceData::new_unchecked(base, multiplier) }; + Array::try_from_parts(ArrayNew::new(Sequence, dtype, length, data).with_stats(stats)) + .vortex_expect("pre-validated SequenceArray parts must be valid") + } + /// Construct a new [`SequenceArray`] from its components. pub fn try_new( base: PValue, @@ -455,13 +421,10 @@ impl Sequence { nullability: Nullability, length: usize, ) -> VortexResult { - Array::try_from_data(SequenceData::try_new( - base, - multiplier, - ptype, - nullability, - length, - )?) + let dtype = DType::Primitive(ptype, nullability); + let data = SequenceData::try_new(base, multiplier, ptype, nullability, length)?; + let stats = Self::stats(data.multiplier()); + Array::try_from_parts(ArrayNew::new(Sequence, dtype, length, data).with_stats(stats)) } /// Construct a new typed [`SequenceArray`] from base/multiplier values. @@ -471,12 +434,16 @@ impl Sequence { nullability: Nullability, length: usize, ) -> VortexResult { - Array::try_from_data(SequenceData::try_new_typed( + let ptype = T::PTYPE; + let dtype = DType::Primitive(ptype, nullability); + let data = SequenceData::try_new_typed( base, multiplier, nullability, length, - )?) + )?; + let stats = Self::stats(data.multiplier()); + Array::try_from_parts(ArrayNew::new(Sequence, dtype, length, data).with_stats(stats)) } } diff --git a/encodings/sequence/src/compress.rs b/encodings/sequence/src/compress.rs index b073ace8dc2..6321aa3df78 100644 --- a/encodings/sequence/src/compress.rs +++ b/encodings/sequence/src/compress.rs @@ -18,9 +18,10 @@ use vortex_buffer::BufferMut; use vortex_buffer::trusted_len::TrustedLen; use vortex_error::VortexResult; +use crate::Sequence; use crate::SequenceArray; -/// An iterator that yields `base, base + step, base + 2*step, ...` via repeated addition. use crate::SequenceData; +/// An iterator that yields `base, base + step, base + 2*step, ...` via repeated addition. struct SequenceIter { acc: T, step: T, @@ -112,7 +113,7 @@ fn encode_primitive_array + CheckedAdd + CheckedSu ) -> VortexResult> { if slice.len() == 1 { // The multiplier here can be any value, zero is chosen - return SequenceData::try_new_typed(slice[0], P::zero(), nullability, 1) + return Sequence::try_new_typed(slice[0], P::zero(), nullability, 1) .map(|a| Some(a.into_array())); } let base = slice[0]; @@ -133,7 +134,7 @@ fn encode_primitive_array + CheckedAdd + CheckedSu .windows(2) .all(|w| Some(w[1]) == w[0].checked_add(&multiplier)) .then_some( - SequenceData::try_new_typed(base, multiplier, nullability, slice.len()) + Sequence::try_new_typed(base, multiplier, nullability, slice.len()) .map(|a| a.into_array()), ) .transpose() diff --git a/encodings/sequence/src/compute/cast.rs b/encodings/sequence/src/compute/cast.rs index 54f3393f16e..1eb9ddf47d8 100644 --- a/encodings/sequence/src/compute/cast.rs +++ b/encodings/sequence/src/compute/cast.rs @@ -13,7 +13,6 @@ use vortex_error::VortexResult; use vortex_error::vortex_err; use crate::Sequence; -use crate::SequenceData; impl CastReduce for Sequence { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // SequenceArray represents arithmetic sequences (base + i * multiplier) which @@ -32,7 +31,7 @@ impl CastReduce for Sequence { // For SequenceArray, we can just create a new one with the same parameters // but different nullability return Ok(Some( - SequenceData::try_new( + Sequence::try_new( array.base(), array.multiplier(), *target_ptype, @@ -71,7 +70,7 @@ impl CastReduce for Sequence { .ok_or_else(|| vortex_err!("Cast resulted in null multiplier value"))?; return Ok(Some( - SequenceData::try_new( + Sequence::try_new( new_base, new_multiplier, *target_ptype, diff --git a/encodings/sequence/src/compute/min_max.rs b/encodings/sequence/src/compute/min_max.rs index 17441bbeb4b..67a7a4e6ebb 100644 --- a/encodings/sequence/src/compute/min_max.rs +++ b/encodings/sequence/src/compute/min_max.rs @@ -16,6 +16,7 @@ use vortex_array::scalar::ScalarValue; use vortex_error::VortexResult; use crate::Sequence; +use crate::SequenceData; /// Sequence-specific min/max kernel. /// @@ -47,7 +48,8 @@ impl DynAggregateKernel for SequenceMinMaxKernel { } let base = seq.base(); - let last = seq.last(); + let last = SequenceData::try_last(base, seq.multiplier(), seq.ptype(), seq.len()) + .expect("validated array"); // Determine min and max based on multiplier direction. // For unsigned types, multiplier is always >= 0. diff --git a/encodings/sequence/src/compute/slice.rs b/encodings/sequence/src/compute/slice.rs index 3eb79f0b23b..c2b64b68ef4 100644 --- a/encodings/sequence/src/compute/slice.rs +++ b/encodings/sequence/src/compute/slice.rs @@ -10,14 +10,13 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::Sequence; -use crate::SequenceData; impl SliceReduce for Sequence { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { // SAFETY: this is a slice of an already-validated `SequenceArray`, so this is still valid. Ok(Some( unsafe { - SequenceData::new_unchecked( + Sequence::new_unchecked( array.index_value(range.start), array.multiplier(), array.ptype(), diff --git a/encodings/sparse/src/canonical.rs b/encodings/sparse/src/canonical.rs index a3cb566dddb..9dbf83b4025 100644 --- a/encodings/sparse/src/canonical.rs +++ b/encodings/sparse/src/canonical.rs @@ -5,10 +5,13 @@ use std::sync::Arc; use itertools::Itertools; use num_traits::NumCast; +use vortex_array::Array; +use vortex_array::ArrayNew; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::BoolArray; +use vortex_array::arrays::Decimal; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::ListViewArray; use vortex_array::arrays::NullArray; @@ -52,8 +55,8 @@ use vortex_error::vortex_bail; use vortex_error::vortex_panic; use crate::ConstantArray; +use crate::Sparse; use crate::SparseArray; -use crate::SparseData; pub(super) fn execute_sparse( array: &SparseArray, ctx: &mut ExecutionCtx, @@ -437,7 +440,7 @@ fn execute_sparse_struct( .cloned() .zip_eq(fill_values) .map(|(patch_values, fill_value)| unsafe { - SparseData::new_unchecked( + Sparse::new_unchecked( unresolved_patches .clone() .map_values(|_| Ok(patch_values)) @@ -474,8 +477,10 @@ fn execute_sparse_decimal( } } let filled_array = builder.finish_into_decimal(); - let array = filled_array.into_data().patch(patches, ctx)?; - Ok(array.into_array()) + let dtype = filled_array.dtype().clone(); + let len = filled_array.len(); + let data = filled_array.into_data().patch(patches, ctx)?; + Ok(Array::try_from_parts(ArrayNew::new(Decimal, dtype, len, data))?.into_array()) } fn execute_varbin( diff --git a/encodings/sparse/src/compute/cast.rs b/encodings/sparse/src/compute/cast.rs index 9806974979a..a9fe0046646 100644 --- a/encodings/sparse/src/compute/cast.rs +++ b/encodings/sparse/src/compute/cast.rs @@ -11,7 +11,6 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::Sparse; -use crate::SparseData; impl CastReduce for Sparse { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { let casted_patches = array @@ -28,7 +27,7 @@ impl CastReduce for Sparse { }; Ok(Some( - SparseData::try_new_from_patches(casted_patches, casted_fill)?.into_array(), + Sparse::try_new_from_patches(casted_patches, casted_fill)?.into_array(), )) } } diff --git a/encodings/sparse/src/compute/filter.rs b/encodings/sparse/src/compute/filter.rs index a3d3d5a1946..f804119dbf3 100644 --- a/encodings/sparse/src/compute/filter.rs +++ b/encodings/sparse/src/compute/filter.rs @@ -11,7 +11,6 @@ use vortex_mask::Mask; use crate::ConstantArray; use crate::Sparse; -use crate::SparseData; impl FilterKernel for Sparse { fn filter( array: ArrayView<'_, Self>, @@ -27,7 +26,7 @@ impl FilterKernel for Sparse { }; Ok(Some( - SparseData::try_new_from_patches(new_patches, array.fill_scalar().clone())? + Sparse::try_new_from_patches(new_patches, array.fill_scalar().clone())? .into_array(), )) } diff --git a/encodings/sparse/src/compute/take.rs b/encodings/sparse/src/compute/take.rs index ebcc2570fc1..b0b5f83cad0 100644 --- a/encodings/sparse/src/compute/take.rs +++ b/encodings/sparse/src/compute/take.rs @@ -10,7 +10,6 @@ use vortex_error::VortexResult; use crate::ConstantArray; use crate::Sparse; -use crate::SparseData; impl TakeExecute for Sparse { fn take( array: ArrayView<'_, Self>, @@ -40,7 +39,7 @@ impl TakeExecute for Sparse { } Ok(Some( - SparseData::try_new_from_patches( + Sparse::try_new_from_patches( new_patches, array.fill_scalar().cast( &array diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index 2d79238230b..6d20fa2e461 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -7,6 +7,7 @@ use std::hash::Hash; use kernel::PARENT_KERNELS; use prost::Message as _; use vortex_array::Array; +use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; @@ -28,7 +29,6 @@ use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; use vortex_array::scalar_fn::fns::operators::Operator; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; @@ -57,12 +57,6 @@ mod slice; vtable!(Sparse, Sparse, SparseData); -#[derive(Debug)] -pub struct SparseMetadata { - patches: PatchesMetadata, - fill_value: Scalar, -} - #[derive(Clone, prost::Message)] #[repr(C)] pub struct ProstPatchesMetadata { @@ -73,28 +67,15 @@ pub struct ProstPatchesMetadata { impl VTable for Sparse { type ArrayData = SparseData; - type Metadata = SparseMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Sparse - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &SparseData) -> usize { - array.patches.array_len() - } - - fn dtype(array: &SparseData) -> &DType { - array.fill_scalar().dtype() - } - - fn stats(array: &SparseData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + SparseData::validate(data.patches(), data.fill_scalar(), dtype, len) } fn array_hash(array: &SparseData, state: &mut H, precision: Precision) { @@ -128,18 +109,10 @@ impl VTable for Sparse { } } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { let patches = array.patches().to_metadata(array.len(), array.dtype())?; - - Ok(SparseMetadata { - patches, - fill_value: array.fill_value.clone(), - }) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { let prost_patches = ProstPatchesMetadata { - patches: metadata.patches, + patches, }; // Note that we DO NOT serialize the fill value since that is stored in the buffers. @@ -147,13 +120,15 @@ impl VTable for Sparse { } fn deserialize( - bytes: &[u8], + &self, dtype: &DType, - _len: usize, + len: usize, + metadata: &[u8], buffers: &[BufferHandle], + children: &dyn ArrayChildren, session: &VortexSession, - ) -> VortexResult { - let prost_patches = ProstPatchesMetadata::decode(bytes)?; + ) -> VortexResult { + let prost_patches = ProstPatchesMetadata::decode(metadata)?; // Once we have the patches metadata, we need to get the fill value from the buffers. @@ -165,19 +140,6 @@ impl VTable for Sparse { let scalar_value = ScalarValue::from_proto_bytes(scalar_bytes, dtype, session)?; let fill_value = Scalar::try_new(dtype.clone(), scalar_value)?; - Ok(SparseMetadata { - patches: prost_patches.patches, - fill_value, - }) - } - - fn build( - dtype: &DType, - len: usize, - metadata: &Self::Metadata, - _buffers: &[BufferHandle], - children: &dyn ArrayChildren, - ) -> VortexResult { vortex_ensure_eq!( children.len(), 2, @@ -187,20 +149,20 @@ impl VTable for Sparse { let patch_indices = children.get( 0, - &metadata.patches.indices_dtype()?, - metadata.patches.len()?, + &prost_patches.patches.indices_dtype()?, + prost_patches.patches.len()?, )?; - let patch_values = children.get(1, dtype, metadata.patches.len()?)?; + let patch_values = children.get(1, dtype, prost_patches.patches.len()?)?; SparseData::try_new_from_patches( Patches::new( len, - metadata.patches.offset()?, + prost_patches.patches.offset()?, patch_indices, patch_values, None, )?, - metadata.fill_value.clone(), + fill_value, ) } @@ -276,7 +238,6 @@ pub struct SparseData { pub(crate) slots: Vec>, patches: Patches, fill_value: Scalar, - stats_set: ArrayStats, } #[derive(Clone, Debug)] @@ -292,7 +253,24 @@ impl Sparse { len: usize, fill_value: Scalar, ) -> VortexResult { - Array::try_from_data(SparseData::try_new(indices, values, len, fill_value)?) + let dtype = fill_value.dtype().clone(); + let data = SparseData::try_new(indices, values, len, fill_value)?; + Array::try_from_parts(ArrayNew::new(Sparse, dtype, len, data)) + } + + pub fn try_new_from_patches(patches: Patches, fill_value: Scalar) -> VortexResult { + let dtype = fill_value.dtype().clone(); + let len = patches.array_len(); + let data = SparseData::try_new_from_patches(patches, fill_value)?; + Array::try_from_parts(ArrayNew::new(Sparse, dtype, len, data)) + } + + pub(crate) unsafe fn new_unchecked(patches: Patches, fill_value: Scalar) -> SparseArray { + let dtype = fill_value.dtype().clone(); + let len = patches.array_len(); + let data = unsafe { SparseData::new_unchecked(patches, fill_value) }; + Array::try_from_parts(ArrayNew::new(Sparse, dtype, len, data)) + .vortex_expect("pre-validated SparseArray parts must be valid") } /// Encode the given array as a [`SparseArray`]. @@ -302,6 +280,33 @@ impl Sparse { } impl SparseData { + pub fn validate( + patches: &Patches, + fill_value: &Scalar, + dtype: &DType, + len: usize, + ) -> VortexResult<()> { + vortex_ensure!( + fill_value.dtype() == dtype, + "fill value dtype {} does not match array dtype {}", + fill_value.dtype(), + dtype, + ); + vortex_ensure!( + patches.array_len() == len, + "patches length {} does not match array length {}", + patches.array_len(), + len + ); + vortex_ensure!( + patches.values().dtype() == dtype, + "patch values dtype {} does not match array dtype {}", + patches.values().dtype(), + dtype, + ); + Ok(()) + } + fn make_slots(patches: &Patches) -> Vec> { vec![ Some(patches.indices().clone()), @@ -343,14 +348,7 @@ impl SparseData { // TODO(0ax1): handle chunk offsets let patches = Patches::new(len, 0, indices, values, None)?; - let slots = Self::make_slots(&patches); - - Ok(Self { - slots, - patches, - fill_value, - stats_set: Default::default(), - }) + Self::try_new_from_patches(patches, fill_value) } /// Build a new SparseArray from an existing set of patches. @@ -369,7 +367,6 @@ impl SparseData { slots, patches, fill_value, - stats_set: Default::default(), }) } @@ -380,7 +377,6 @@ impl SparseData { slots, patches, fill_value, - stats_set: Default::default(), } } @@ -474,13 +470,13 @@ impl SparseData { } }; - return Ok(SparseData::try_new( + return Sparse::try_new( non_null_indices, non_null_values, array.len(), Scalar::null(array.dtype().clone()), - )? - .into_array()); + ) + .map(IntoArray::into_array); } let fill = if let Some(fill) = fill_value { @@ -521,8 +517,8 @@ impl SparseData { Mask::Values(values) => values.indices().iter().map(|v| *v as u64).collect(), }; - SparseData::try_new(indices.into_array(), non_top_values, array.len(), fill) - .map(|a| a.into_array()) + Sparse::try_new(indices.into_array(), non_top_values, array.len(), fill) + .map(IntoArray::into_array) } } @@ -544,7 +540,7 @@ impl ValidityVTable for Sparse { }; Ok(Validity::Array( - unsafe { SparseData::new_unchecked(patches, array.fill_value.is_valid().into()) } + unsafe { Sparse::new_unchecked(patches, array.fill_value.is_valid().into()) } .into_array(), )) } diff --git a/encodings/sparse/src/rules.rs b/encodings/sparse/src/rules.rs index 91b4edf80b0..f9c2832cb63 100644 --- a/encodings/sparse/src/rules.rs +++ b/encodings/sparse/src/rules.rs @@ -12,7 +12,6 @@ use vortex_array::scalar_fn::fns::not::NotReduceAdaptor; use vortex_error::VortexResult; use crate::Sparse; -use crate::SparseData; pub(crate) static RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&CastReduceAdaptor(Sparse)), @@ -23,8 +22,6 @@ impl NotReduce for Sparse { fn invert(array: ArrayView<'_, Self>) -> VortexResult> { let inverted_fill = array.fill_scalar().as_bool().invert().into_scalar(); let inverted_patches = array.patches().clone().map_values(|values| values.not())?; - Ok(Some( - SparseData::try_new_from_patches(inverted_patches, inverted_fill)?.into_array(), - )) + Ok(Some(Sparse::try_new_from_patches(inverted_patches, inverted_fill)?.into_array())) } } diff --git a/encodings/sparse/src/slice.rs b/encodings/sparse/src/slice.rs index 6d06da58029..9952954a229 100644 --- a/encodings/sparse/src/slice.rs +++ b/encodings/sparse/src/slice.rs @@ -12,7 +12,6 @@ use vortex_error::VortexResult; use crate::ConstantArray; use crate::Sparse; -use crate::SparseData; impl SliceKernel for Sparse { fn slice( @@ -36,7 +35,7 @@ impl SliceKernel for Sparse { // patches slice will ensure that dtype of patches is unchanged and the indices and // values match Ok(Some( - unsafe { SparseData::new_unchecked(new_patches, array.fill_scalar().clone()) } + unsafe { Sparse::new_unchecked(new_patches, array.fill_scalar().clone()) } .into_array(), )) } diff --git a/encodings/zigzag/src/array.rs b/encodings/zigzag/src/array.rs index 0cb2102c8c6..099bedb0306 100644 --- a/encodings/zigzag/src/array.rs +++ b/encodings/zigzag/src/array.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::Array; +use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; @@ -18,7 +19,6 @@ use vortex_array::dtype::PType; use vortex_array::match_each_unsigned_integer_ptype; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; use vortex_array::vtable::VTable; @@ -42,28 +42,18 @@ vtable!(ZigZag, ZigZag, ZigZagData); impl VTable for ZigZag { type ArrayData = ZigZagData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; - fn vtable(_array: &Self::ArrayData) -> &Self { - &ZigZag - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ZigZagData) -> usize { - array.encoded().len() - } - - fn dtype(array: &ZigZagData) -> &DType { - &array.dtype - } - - fn stats(array: &ZigZagData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + let expected_dtype = ZigZagData::dtype_from_encoded_dtype(data.encoded().dtype())?; + vortex_ensure!(dtype == &expected_dtype, "expected dtype {expected_dtype}, got {dtype}"); + vortex_ensure!(data.encoded().len() == len, "expected len {len}, got {}", data.encoded().len()); + Ok(()) } fn array_hash(array: &ZigZagData, state: &mut H, precision: Precision) { @@ -86,31 +76,20 @@ impl VTable for ZigZag { vortex_panic!("ZigZagArray buffer_name index {idx} out of bounds") } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) - } - - fn build( + &self, dtype: &DType, len: usize, - _metadata: &Self::Metadata, + metadata: &[u8], _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + ::deserialize(metadata)?; if children.len() != 1 { vortex_bail!("Expected 1 child, got {}", children.len()); } @@ -172,9 +151,7 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["encoded"]; #[derive(Clone, Debug)] pub struct ZigZagData { - dtype: DType, pub(super) slots: Vec>, - stats_set: ArrayStats, } #[derive(Clone, Debug)] @@ -185,11 +162,19 @@ impl ZigZag { /// Construct a new [`ZigZagArray`] from an encoded unsigned integer array. pub fn try_new(encoded: ArrayRef) -> VortexResult { - Array::try_from_data(ZigZagData::try_new(encoded)?) + let data = ZigZagData::try_new(encoded)?; + let dtype = ZigZagData::dtype_from_encoded_dtype(data.encoded().dtype())?; + let len = data.len(); + Array::try_from_parts(ArrayNew::new(ZigZag, dtype, len, data)) } } impl ZigZagData { + fn dtype_from_encoded_dtype(encoded_dtype: &DType) -> VortexResult { + Ok(DType::from(PType::try_from(encoded_dtype)?.to_signed()) + .with_nullability(encoded_dtype.nullability())) + } + pub fn new(encoded: ArrayRef) -> Self { Self::try_new(encoded).vortex_expect("ZigZagArray new") } @@ -200,14 +185,9 @@ impl ZigZagData { vortex_bail!(MismatchedTypes: "unsigned int", encoded_dtype); } - let dtype = DType::from(PType::try_from(&encoded_dtype)?.to_signed()) - .with_nullability(encoded_dtype.nullability()); + Self::dtype_from_encoded_dtype(&encoded_dtype)?; - Ok(Self { - dtype, - slots: vec![Some(encoded)], - stats_set: Default::default(), - }) + Ok(Self { slots: vec![Some(encoded)] }) } /// Returns the length of the array. @@ -222,14 +202,10 @@ impl ZigZagData { self.encoded().is_empty() } - /// Returns the logical data type of the array. - #[inline] - pub fn dtype(&self) -> &DType { - &self.dtype - } - pub fn ptype(&self) -> PType { - self.dtype().as_ptype() + PType::try_from(self.encoded().dtype()) + .vortex_expect("ZigZagArray encoded dtype") + .to_signed() } pub fn encoded(&self) -> &ArrayRef { diff --git a/encodings/zigzag/src/compress.rs b/encodings/zigzag/src/compress.rs index 690cd52deac..b2d142c755c 100644 --- a/encodings/zigzag/src/compress.rs +++ b/encodings/zigzag/src/compress.rs @@ -12,8 +12,8 @@ use vortex_error::vortex_bail; use vortex_error::vortex_panic; use zigzag::ZigZag as ExternalZigZag; +use crate::ZigZag; use crate::ZigZagArray; -use crate::ZigZagData; pub fn zigzag_encode(parray: PrimitiveArray) -> VortexResult { let validity = parray.validity(); let encoded = match parray.ptype() { @@ -26,7 +26,7 @@ pub fn zigzag_encode(parray: PrimitiveArray) -> VortexResult { parray.ptype() ), }; - ZigZagArray::try_from_data(ZigZagData::try_new(encoded.into_array())?) + ZigZag::try_new(encoded.into_array()) } fn zigzag_encode_primitive( diff --git a/encodings/zigzag/src/compute/cast.rs b/encodings/zigzag/src/compute/cast.rs index c69d91f5347..138c6aae736 100644 --- a/encodings/zigzag/src/compute/cast.rs +++ b/encodings/zigzag/src/compute/cast.rs @@ -10,7 +10,6 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::ZigZag; -use crate::ZigZagData; impl CastReduce for ZigZag { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { if !dtype.is_signed_int() { @@ -20,7 +19,7 @@ impl CastReduce for ZigZag { let new_encoded_dtype = DType::Primitive(dtype.as_ptype().to_unsigned(), dtype.nullability()); let new_encoded = array.encoded().cast(new_encoded_dtype)?; - Ok(Some(ZigZagData::try_new(new_encoded)?.into_array())) + Ok(Some(ZigZag::try_new(new_encoded)?.into_array())) } } diff --git a/encodings/zigzag/src/compute/mod.rs b/encodings/zigzag/src/compute/mod.rs index 9967ba5bb95..11fe580e6bc 100644 --- a/encodings/zigzag/src/compute/mod.rs +++ b/encodings/zigzag/src/compute/mod.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use crate::ZigZagData; mod cast; use vortex_array::ArrayRef; @@ -22,7 +21,7 @@ use crate::ZigZag; impl FilterReduce for ZigZag { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { let encoded = array.encoded().filter(mask.clone())?; - Ok(Some(ZigZagData::try_new(encoded)?.into_array())) + Ok(Some(ZigZag::try_new(encoded)?.into_array())) } } @@ -33,7 +32,7 @@ impl TakeExecute for ZigZag { _ctx: &mut ExecutionCtx, ) -> VortexResult> { let encoded = array.encoded().take(indices.clone())?; - Ok(Some(ZigZagData::try_new(encoded)?.into_array())) + Ok(Some(ZigZag::try_new(encoded)?.into_array())) } } @@ -44,7 +43,7 @@ impl MaskReduce for ZigZag { EmptyOptions, [array.encoded().clone(), mask.clone()], )?; - Ok(Some(ZigZagData::try_new(masked_encoded)?.into_array())) + Ok(Some(ZigZag::try_new(masked_encoded)?.into_array())) } } diff --git a/encodings/zigzag/src/slice.rs b/encodings/zigzag/src/slice.rs index 599d365809f..26bc92b820b 100644 --- a/encodings/zigzag/src/slice.rs +++ b/encodings/zigzag/src/slice.rs @@ -10,12 +10,9 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::ZigZag; -use crate::ZigZagData; impl SliceReduce for ZigZag { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - Ok(Some( - ZigZagData::new(array.encoded().slice(range)?).into_array(), - )) + Ok(Some(ZigZag::try_new(array.encoded().slice(range)?)?.into_array())) } } diff --git a/vortex-array/src/array/erased.rs b/vortex-array/src/array/erased.rs index bdb67697c3a..f2c1a7a50d5 100644 --- a/vortex-array/src/array/erased.rs +++ b/vortex-array/src/array/erased.rs @@ -467,11 +467,6 @@ impl ArrayRef { self.0.metadata(self) } - /// Formats a human-readable metadata description. - pub fn metadata_fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - self.0.metadata_fmt(self, f) - } - /// Returns whether all buffers are host-resident. pub fn is_host(&self) -> bool { for array in self.depth_first_traversal() { diff --git a/vortex-array/src/array/mod.rs b/vortex-array/src/array/mod.rs index 842f759ace7..91701e6fae2 100644 --- a/vortex-array/src/array/mod.rs +++ b/vortex-array/src/array/mod.rs @@ -3,7 +3,6 @@ use std::any::Any; use std::fmt::Debug; -use std::fmt::Formatter; use std::hash::Hash; use std::hash::Hasher; @@ -122,9 +121,6 @@ pub(crate) trait DynArray: 'static + private::Sealed + Send + Sync + Debug { /// support serialization. fn metadata(&self, this: &ArrayRef) -> VortexResult>>; - /// Formats a human-readable metadata description. - fn metadata_fmt(&self, this: &ArrayRef, f: &mut Formatter<'_>) -> std::fmt::Result; - /// Hashes the array contents including len, dtype, and encoding id. fn dyn_array_hash(&self, state: &mut dyn Hasher, precision: crate::Precision); @@ -304,15 +300,7 @@ impl DynArray for ArrayInner { fn metadata(&self, this: &ArrayRef) -> VortexResult>> { let view = unsafe { ArrayView::new_unchecked(this, &self.data) }; - V::serialize(V::metadata(view)?) - } - - fn metadata_fmt(&self, this: &ArrayRef, f: &mut Formatter<'_>) -> std::fmt::Result { - let view = unsafe { ArrayView::new_unchecked(this, &self.data) }; - match V::metadata(view) { - Err(e) => write!(f, ""), - Ok(metadata) => Debug::fmt(&metadata, f), - } + V::serialize(view) } fn dyn_array_hash(&self, state: &mut dyn Hasher, precision: crate::Precision) { diff --git a/vortex-array/src/array/typed.rs b/vortex-array/src/array/typed.rs index e635a98227d..c2f3dc74583 100644 --- a/vortex-array/src/array/typed.rs +++ b/vortex-array/src/array/typed.rs @@ -24,6 +24,33 @@ use crate::dtype::DType; use crate::stats::ArrayStats; use crate::stats::StatsSetRef; use crate::validity::Validity; + +/// Construction parameters for [`ArrayInner`] / [`Array`]. +pub struct ArrayNew { + pub vtable: V, + pub dtype: DType, + pub len: usize, + pub data: V::ArrayData, + pub stats: ArrayStats, +} + +impl ArrayNew { + pub fn new(vtable: V, dtype: DType, len: usize, data: V::ArrayData) -> Self { + Self { + vtable, + dtype, + len, + data, + stats: ArrayStats::default(), + } + } + + #[must_use] + pub fn with_stats(mut self, stats: ArrayStats) -> Self { + self.stats = stats; + self + } +} // ============================================================================= // ArrayInner — the concrete type stored inside Arc // ============================================================================= @@ -43,14 +70,11 @@ pub(crate) struct ArrayInner { } impl ArrayInner { - /// Create a new inner array from encoding-specific data. + /// Create a new inner array from explicit construction parameters. #[doc(hidden)] - pub fn try_from_data(data: V::ArrayData) -> VortexResult { - let vtable = V::vtable(&data).clone(); - let dtype = V::dtype(&data).clone(); - let len = V::len(&data); - let stats = V::stats(&data).clone(); - Ok(unsafe { Self::from_data_unchecked(vtable, dtype, len, data, stats) }) + pub fn try_new(new: ArrayNew) -> VortexResult { + new.vtable.validate(&new.data, &new.dtype, new.len)?; + Ok(unsafe { Self::from_data_unchecked(new.vtable, new.dtype, new.len, new.data, new.stats) }) } /// Create without validation. @@ -140,15 +164,35 @@ pub struct Array { #[allow(clippy::same_name_method)] impl Array { - /// Create a typed array from encoding-specific data. - pub fn try_from_data(data: V::ArrayData) -> VortexResult { - let inner = ArrayRef::from_inner(Arc::new(ArrayInner::::try_from_data(data)?)); + /// Create a typed array from explicit construction parameters. + pub fn try_from_parts(new: ArrayNew) -> VortexResult { + let inner = ArrayRef::from_inner(Arc::new(ArrayInner::::try_new(new)?)); Ok(Self { inner, _phantom: PhantomData, }) } + /// Create a typed array from explicit construction parameters without validation. + /// + /// # Safety + /// Caller must ensure the provided parts are logically consistent. + pub(crate) unsafe fn from_parts_unchecked(new: ArrayNew) -> Self { + let inner = ArrayRef::from_inner(Arc::new(unsafe { + ArrayInner::::from_data_unchecked( + new.vtable, + new.dtype, + new.len, + new.data, + new.stats, + ) + })); + Self { + inner, + _phantom: PhantomData, + } + } + /// Create from an existing `ArrayRef`, trusting that it contains `ArrayInner`. /// /// # Safety diff --git a/vortex-array/src/array/vtable/dyn_.rs b/vortex-array/src/array/vtable/dyn_.rs index ca6cbaf0cb4..458870c825a 100644 --- a/vortex-array/src/array/vtable/dyn_.rs +++ b/vortex-array/src/array/vtable/dyn_.rs @@ -15,8 +15,8 @@ use crate::ExecutionResult; use crate::ExecutionStep; use crate::IntoArray; use crate::array::Array; +use crate::array::ArrayNew; use crate::array::ArrayId; -use crate::array::ArrayInner; use crate::array::VTable; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -85,27 +85,12 @@ impl DynVTable for V { children: &dyn ArrayChildren, session: &VortexSession, ) -> VortexResult { - let metadata = V::deserialize(metadata, dtype, len, buffers, session)?; - let inner = V::build(dtype, len, &metadata, buffers, children)?; - // Validate the inner array's properties before wrapping. - assert_eq!(V::len(&inner), len, "Array length mismatch after building"); - assert_eq!( - V::dtype(&inner), - dtype, - "Array dtype mismatch after building" - ); - // Wrap in ArrayInner for safe downcasting. - // SAFETY: We just validated that V::len(&inner) == len and V::dtype(&inner) == dtype. - let array = unsafe { - ArrayInner::from_data_unchecked( - self.clone(), - dtype.clone(), - len, - inner, - ArrayStats::default(), - ) - }; - Ok(array.into_array()) + let inner = self.deserialize(dtype, len, metadata, buffers, children, session)?; + Ok(Array::::try_from_parts( + ArrayNew::new(self.clone(), dtype.clone(), len, inner) + .with_stats(ArrayStats::default()), + )? + .into_array()) } fn with_slots(&self, array: ArrayRef, slots: Vec>) -> VortexResult { @@ -114,7 +99,13 @@ impl DynVTable for V { .vortex_expect("Failed to downcast array"); let mut data = typed.data().clone(); V::with_slots(&mut data, slots)?; - Ok(Array::::try_from_data(data)?.into_array()) + Ok(unsafe { + Array::::from_parts_unchecked( + ArrayNew::new(self.clone(), array.dtype().clone(), array.len(), data) + .with_stats(array.statistics().to_array_stats()), + ) + .into_array() + }) } fn reduce(&self, array: &ArrayRef) -> VortexResult> { diff --git a/vortex-array/src/array/vtable/mod.rs b/vortex-array/src/array/vtable/mod.rs index 7f233b72f5d..8a2f258697b 100644 --- a/vortex-array/src/array/vtable/mod.rs +++ b/vortex-array/src/array/vtable/mod.rs @@ -35,7 +35,6 @@ use crate::executor::ExecutionCtx; use crate::patches::Patches; use crate::scalar::ScalarValue; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; /// The array [`VTable`] encapsulates logic for an Array type within Vortex. @@ -52,28 +51,18 @@ use crate::validity::Validity; /// out of bounds). Post-conditions are validated after invocation of the vtable function and will /// panic if violated. pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { - type ArrayData: 'static + Send + Sync + Clone + Debug + IntoArray; - type Metadata: Debug; + type ArrayData: 'static + Send + Sync + Clone + Debug; type OperationsVTable: OperationsVTable; type ValidityVTable: ValidityVTable; - /// Returns the VTable from the array instance. - /// - // NOTE(ngates): this function is temporary while we migrate Arrays over to the unified vtable - fn vtable(array: &Self::ArrayData) -> &Self; - /// Returns the ID of the array. fn id(&self) -> ArrayId; - /// Returns the length of the array. - fn len(array: &Self::ArrayData) -> usize; - - /// Returns the DType of the array. - fn dtype(array: &Self::ArrayData) -> &DType; - - /// Returns the stats set for the array. - fn stats(array: &Self::ArrayData) -> &ArrayStats; + /// Validates that externally supplied logical metadata matches the array data. + fn validate(&self, _data: &Self::ArrayData, _dtype: &DType, _len: usize) -> VortexResult<()> { + Ok(()) + } /// Hashes the array contents. fn array_hash(array: &Self::ArrayData, state: &mut H, precision: Precision); @@ -130,21 +119,20 @@ pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { .vortex_expect("child_name index out of bounds") } - /// Exports metadata for an array. - fn metadata(array: ArrayView<'_, Self>) -> VortexResult; - /// Serialize metadata into a byte buffer for IPC or file storage. /// Return `None` if the array cannot be serialized. - fn serialize(metadata: Self::Metadata) -> VortexResult>>; + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>>; - /// Deserialize array metadata from a byte buffer. + /// Deserialize an array from serialized components. fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult; + &self, + dtype: &DType, + len: usize, + metadata: &[u8], + buffers: &[BufferHandle], + children: &dyn ArrayChildren, + session: &VortexSession, + ) -> VortexResult; /// Writes the array into a canonical builder. fn append_to_builder( @@ -161,15 +149,6 @@ pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { Ok(()) } - /// Build an array from components. - fn build( - dtype: &DType, - len: usize, - metadata: &Self::Metadata, - buffers: &[BufferHandle], - children: &dyn ArrayChildren, - ) -> VortexResult; - /// Returns the slots of the array as a slice. /// /// Slots provide fixed-position storage for child arrays. Each encoding defines named @@ -343,23 +322,12 @@ macro_rules! vtable { // Legacy form: FooArray is the inner struct name, no type alias generated. ($Base:ident, $VT:ident) => { $crate::aliases::paste::paste! { - impl $crate::IntoArray for [<$Base Array>] { - fn into_array(self) -> $crate::ArrayRef { - use $crate::aliases::vortex_error::VortexExpect; - $crate::ArrayRef::from($crate::Array::<$VT>::try_from_data(self).vortex_expect("data is always valid")) - } - } - - impl From<[<$Base Array>]> for $crate::ArrayRef { - fn from(value: [<$Base Array>]) -> $crate::ArrayRef { - use $crate::IntoArray; - value.into_array() - } - } - impl [<$Base Array>] { #[deprecated(note = "use `.into_array()` (owned) or `.clone().into_array()` (ref) to make clones explicit")] - pub fn to_array(&self) -> $crate::ArrayRef { + pub fn to_array(&self) -> $crate::ArrayRef + where + Self: Clone + $crate::IntoArray, + { use $crate::IntoArray; self.clone().into_array() } @@ -371,20 +339,6 @@ macro_rules! vtable { $crate::aliases::paste::paste! { /// Type alias: `FooArray = Array`. pub type [<$Base Array>] = $crate::Array<$VT>; - - impl $crate::IntoArray for $Data { - fn into_array(self) -> $crate::ArrayRef { - use $crate::aliases::vortex_error::VortexExpect; - $crate::Array::<$VT>::try_from_data(self).vortex_expect("data is always valid").into_array() - } - } - - impl From<$Data> for $crate::ArrayRef { - fn from(value: $Data) -> $crate::ArrayRef { - use $crate::IntoArray; - value.into_array() - } - } } }; } diff --git a/vortex-array/src/arrays/bool/array.rs b/vortex-array/src/arrays/bool/array.rs index 4c654a44081..56b716e9fcf 100644 --- a/vortex-array/src/arrays/bool/array.rs +++ b/vortex-array/src/arrays/bool/array.rs @@ -12,6 +12,7 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; use crate::array::Array; +use crate::array::ArrayNew; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Bool; @@ -186,7 +187,10 @@ impl Array { /// /// Returns an error if the provided components do not satisfy the invariants. pub fn try_new(bits: BitBuffer, validity: Validity) -> VortexResult { - Array::try_from_data(BoolData::try_new(bits, validity)?) + let data = BoolData::try_new(bits, validity)?; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Bool, dtype, len, data)) } /// Build a new bool array from a `BufferHandle`, returning an error if the offset is @@ -197,7 +201,10 @@ impl Array { len: usize, validity: Validity, ) -> VortexResult { - Array::try_from_data(BoolData::try_new_from_handle(bits, offset, len, validity)?) + let data = BoolData::try_new_from_handle(bits, offset, len, validity)?; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Bool, dtype, len, data)) } /// Creates a new [`BoolArray`] without validation. @@ -207,7 +214,10 @@ impl Array { /// The caller must ensure that the validity length is equal to the bit buffer length. pub unsafe fn new_unchecked(bits: BitBuffer, validity: Validity) -> Self { // SAFETY: caller guarantees validity length equals bit buffer length. - Array::try_from_data(unsafe { BoolData::new_unchecked(bits, validity) }) + let data = unsafe { BoolData::new_unchecked(bits, validity) }; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Bool, dtype, len, data)) .vortex_expect("BoolData is always valid") } diff --git a/vortex-array/src/arrays/bool/vtable/mod.rs b/vortex-array/src/arrays/bool/vtable/mod.rs index 17a87a92cbe..a2608f09011 100644 --- a/vortex-array/src/arrays/bool/vtable/mod.rs +++ b/vortex-array/src/arrays/bool/vtable/mod.rs @@ -50,30 +50,13 @@ pub struct BoolMetadata { impl VTable for Bool { type ArrayData = BoolData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Bool - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &BoolData) -> usize { - array.len - } - - fn dtype(array: &BoolData) -> &DType { - &array.dtype - } - - fn stats(array: &BoolData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &BoolData, state: &mut H, precision: Precision) { array.to_bit_buffer().array_hash(state, precision); array.validity().array_hash(state, precision); @@ -104,35 +87,24 @@ impl VTable for Bool { } } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { assert!(array.offset < 8, "Offset must be <8, got {}", array.offset); - Ok(ProstMetadata(BoolMetadata { + Ok(Some(ProstMetadata(BoolMetadata { offset: u32::try_from(array.offset).vortex_expect("checked"), - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + }) + .serialize())) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - let metadata = ::deserialize(bytes)?; - Ok(ProstMetadata(metadata)) - } - - fn build( + &self, dtype: &DType, - len: usize, - metadata: &Self::Metadata, + len: usize, metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = ProstMetadata::::deserialize(metadata)?; if buffers.len() != 1 { vortex_bail!("Expected 1 buffer, got {}", buffers.len()); } diff --git a/vortex-array/src/arrays/chunked/array.rs b/vortex-array/src/arrays/chunked/array.rs index 5aa15ce6d57..80661fb03bd 100644 --- a/vortex-array/src/arrays/chunked/array.rs +++ b/vortex-array/src/arrays/chunked/array.rs @@ -17,6 +17,7 @@ use vortex_error::vortex_bail; use crate::ArrayRef; use crate::IntoArray; use crate::array::Array; +use crate::array::ArrayNew; use crate::arrays::Chunked; use crate::arrays::primitive::PrimitiveData; use crate::dtype::DType; @@ -49,7 +50,16 @@ impl ChunkedData { chunks: &[ArrayRef], ) -> Vec> { let mut slots = Vec::with_capacity(1 + chunks.len()); - slots.push(Some(chunk_offsets.clone().into_array())); + slots.push(Some( + Array::try_from_parts(ArrayNew::new( + crate::arrays::Primitive, + chunk_offsets.dtype().clone(), + chunk_offsets.len(), + chunk_offsets.clone(), + )) + .vortex_expect("chunk offsets are always valid") + .into_array(), + )); slots.extend(chunks.iter().map(|c| Some(c.clone()))); slots } @@ -219,7 +229,7 @@ impl ChunkedData { new_chunks.push( // SAFETY: chunks_to_combine contains valid chunks of the same dtype as self. // All chunks are guaranteed to be valid arrays matching self.dtype(). - unsafe { ChunkedData::new_unchecked(chunks_to_combine, self.dtype().clone()) } + unsafe { Array::::new_unchecked(chunks_to_combine, self.dtype().clone()) } .into_array() .to_canonical()? .into_array(), @@ -243,7 +253,7 @@ impl ChunkedData { new_chunks.push( // SAFETY: chunks_to_combine contains valid chunks of the same dtype as self. // All chunks are guaranteed to be valid arrays matching self.dtype(). - unsafe { ChunkedData::new_unchecked(chunks_to_combine, self.dtype().clone()) } + unsafe { Array::::new_unchecked(chunks_to_combine, self.dtype().clone()) } .into_array() .to_canonical()? .into_array(), @@ -259,7 +269,10 @@ impl ChunkedData { impl Array { /// Constructs a new `ChunkedArray`. pub fn try_new(chunks: Vec, dtype: DType) -> VortexResult { - Array::try_from_data(ChunkedData::try_new(chunks, dtype)?) + let data = ChunkedData::try_new(chunks, dtype)?; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Chunked, dtype, len, data)) } /// Creates a new `ChunkedArray` without validation. @@ -268,7 +281,10 @@ impl Array { /// /// See [`ChunkedData::new_unchecked`]. pub unsafe fn new_unchecked(chunks: Vec, dtype: DType) -> Self { - Array::try_from_data(unsafe { ChunkedData::new_unchecked(chunks, dtype) }) + let data = unsafe { ChunkedData::new_unchecked(chunks, dtype) }; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Chunked, dtype, len, data)) .vortex_expect("ChunkedData is always valid") } } diff --git a/vortex-array/src/arrays/chunked/vtable/mod.rs b/vortex-array/src/arrays/chunked/vtable/mod.rs index 3f20f9d56ea..6038296387c 100644 --- a/vortex-array/src/arrays/chunked/vtable/mod.rs +++ b/vortex-array/src/arrays/chunked/vtable/mod.rs @@ -21,6 +21,7 @@ use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; use crate::array::VTable; +use crate::arrays::PrimitiveArray; use crate::arrays::chunked::ChunkedData; use crate::arrays::chunked::array::CHUNK_OFFSETS_SLOT; use crate::arrays::chunked::array::CHUNKS_OFFSET; @@ -54,33 +55,15 @@ impl Chunked { impl VTable for Chunked { type ArrayData = ChunkedData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Chunked - } fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ChunkedData) -> usize { - array.len - } - - fn dtype(array: &ChunkedData) -> &DType { - &array.dtype - } - - fn stats(array: &ChunkedData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &ChunkedData, state: &mut H, precision: Precision) { - array - .chunk_offsets - .clone() + PrimitiveArray::new(array.chunk_offsets.to_buffer::(), Validity::NonNullable) .into_array() .array_hash(state, precision); for chunk in &array.chunks { @@ -89,11 +72,16 @@ impl VTable for Chunked { } fn array_eq(array: &ChunkedData, other: &ChunkedData, precision: Precision) -> bool { - array - .chunk_offsets - .clone() + PrimitiveArray::new(array.chunk_offsets.to_buffer::(), Validity::NonNullable) .into_array() - .array_eq(&other.chunk_offsets.clone().into_array(), precision) + .array_eq( + &PrimitiveArray::new( + other.chunk_offsets.to_buffer::(), + Validity::NonNullable, + ) + .into_array(), + precision, + ) && array.chunks.len() == other.chunks.len() && array .iter_chunks() @@ -113,31 +101,20 @@ impl VTable for Chunked { vortex_panic!("ChunkedArray buffer_name index {idx} out of bounds") } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) - } - - fn build( + &self, dtype: &DType, - _len: usize, - _metadata: &Self::Metadata, + _len: usize, metadata: &[u8], + _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + ::deserialize(metadata)?; if children.is_empty() { vortex_bail!("Chunked array needs at least one child"); } diff --git a/vortex-array/src/arrays/chunked/vtable/validity.rs b/vortex-array/src/arrays/chunked/vtable/validity.rs index 18e429b46b0..a250c98be89 100644 --- a/vortex-array/src/arrays/chunked/vtable/validity.rs +++ b/vortex-array/src/arrays/chunked/vtable/validity.rs @@ -8,7 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::array::ValidityVTable; use crate::arrays::Chunked; -use crate::arrays::chunked::ChunkedData; +use crate::arrays::ChunkedArray; use crate::dtype::DType; use crate::dtype::Nullability; use crate::validity::Validity; @@ -39,7 +39,7 @@ impl ValidityVTable for Chunked { Ok(Validity::Array( unsafe { - ChunkedData::new_unchecked( + ChunkedArray::new_unchecked( validities .into_iter() .zip(array.iter_chunks()) diff --git a/vortex-array/src/arrays/constant/array.rs b/vortex-array/src/arrays/constant/array.rs index ff32124210a..4475e9dbafd 100644 --- a/vortex-array/src/arrays/constant/array.rs +++ b/vortex-array/src/arrays/constant/array.rs @@ -5,6 +5,7 @@ use vortex_error::VortexExpect; use crate::ArrayRef; use crate::array::Array; +use crate::array::ArrayNew; use crate::arrays::Constant; use crate::dtype::DType; use crate::scalar::Scalar; @@ -64,7 +65,10 @@ impl Array { where S: Into, { - Array::try_from_data(ConstantData::new(scalar, len)) + let scalar = scalar.into(); + let dtype = scalar.dtype().clone(); + let data = ConstantData::new(scalar, len); + Array::try_from_parts(ArrayNew::new(Constant, dtype, len, data)) .vortex_expect("ConstantData is always valid") } } diff --git a/vortex-array/src/arrays/constant/compute/take.rs b/vortex-array/src/arrays/constant/compute/take.rs index 4bbd44132e1..96aa71fd9ac 100644 --- a/vortex-array/src/arrays/constant/compute/take.rs +++ b/vortex-array/src/arrays/constant/compute/take.rs @@ -9,9 +9,9 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Constant; use crate::arrays::ConstantArray; +use crate::arrays::MaskedArray; use crate::arrays::dict::TakeReduce; use crate::arrays::dict::TakeReduceAdaptor; -use crate::arrays::masked::MaskedData; use crate::optimizer::rules::ParentRuleSet; use crate::scalar::Scalar; use crate::validity::Validity; @@ -45,7 +45,7 @@ impl TakeReduce for Constant { return Ok(Some(arr)); } - MaskedData::try_new(arr, Validity::from(v.clone()))?.into_array() + MaskedArray::try_new(arr, Validity::from(v.clone()))?.into_array() } }; Ok(Some(result)) diff --git a/vortex-array/src/arrays/constant/vtable/mod.rs b/vortex-array/src/arrays/constant/vtable/mod.rs index d6fd4407635..b714ba0a30a 100644 --- a/vortex-array/src/arrays/constant/vtable/mod.rs +++ b/vortex-array/src/arrays/constant/vtable/mod.rs @@ -57,30 +57,13 @@ impl Constant { impl VTable for Constant { type ArrayData = ConstantData; - type Metadata = Scalar; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Constant - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ConstantData) -> usize { - array.len - } - - fn dtype(array: &ConstantData) -> &DType { - array.scalar.dtype() - } - - fn stats(array: &ConstantData) -> &ArrayStats { - &array.stats_set - } - fn array_hash( array: &ConstantData, state: &mut H, @@ -131,23 +114,21 @@ impl VTable for Constant { Ok(()) } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(array.scalar().clone()) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { // HACK: Because the scalar is stored in the buffers, we do not need to serialize the // metadata at all. Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], + &self, dtype: &DType, - _len: usize, + len: usize, _metadata: &[u8], + buffers: &[BufferHandle], + _children: &dyn ArrayChildren, session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult { vortex_ensure!( buffers.len() == 1, "Expected 1 buffer, got {}", @@ -160,17 +141,7 @@ impl VTable for Constant { let scalar_value = ScalarValue::from_proto_bytes(bytes, dtype, session)?; let scalar = Scalar::try_new(dtype.clone(), scalar_value)?; - Ok(scalar) - } - - fn build( - _dtype: &DType, - len: usize, - metadata: &Self::Metadata, - _buffers: &[BufferHandle], - _children: &dyn ArrayChildren, - ) -> VortexResult { - Ok(ConstantData::new(metadata.clone(), len)) + Ok(ConstantData::new(scalar, len)) } fn reduce_parent( diff --git a/vortex-array/src/arrays/decimal/array.rs b/vortex-array/src/arrays/decimal/array.rs index f8e3ce0c976..35124837d9a 100644 --- a/vortex-array/src/arrays/decimal/array.rs +++ b/vortex-array/src/arrays/decimal/array.rs @@ -15,6 +15,7 @@ use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; use crate::array::Array; +use crate::array::ArrayNew; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Decimal; @@ -450,7 +451,10 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> Self { - Array::try_from_data(DecimalData::new(buffer, decimal_dtype, validity)) + let data = DecimalData::new(buffer, decimal_dtype, validity); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Decimal, dtype, len, data)) .vortex_expect("DecimalData is always valid") } @@ -464,7 +468,10 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> Self { - Array::try_from_data(unsafe { DecimalData::new_unchecked(buffer, decimal_dtype, validity) }) + let data = unsafe { DecimalData::new_unchecked(buffer, decimal_dtype, validity) }; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Decimal, dtype, len, data)) .vortex_expect("DecimalData is always valid") } @@ -474,7 +481,10 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> VortexResult { - Array::try_from_data(DecimalData::try_new(buffer, decimal_dtype, validity)?) + let data = DecimalData::try_new(buffer, decimal_dtype, validity)?; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Decimal, dtype, len, data)) } /// Creates a new [`DecimalArray`] from an iterator of values. @@ -486,7 +496,10 @@ impl Array { iter: I, decimal_dtype: DecimalDType, ) -> Self { - Array::try_from_data(DecimalData::from_iter(iter, decimal_dtype)) + let data = DecimalData::from_iter(iter, decimal_dtype); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Decimal, dtype, len, data)) .vortex_expect("DecimalData is always valid") } @@ -495,7 +508,10 @@ impl Array { iter: I, decimal_dtype: DecimalDType, ) -> Self { - Array::try_from_data(DecimalData::from_option_iter(iter, decimal_dtype)) + let data = DecimalData::from_option_iter(iter, decimal_dtype); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Decimal, dtype, len, data)) .vortex_expect("DecimalData is always valid") } @@ -506,12 +522,10 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> Self { - Array::try_from_data(DecimalData::new_handle( - values, - values_type, - decimal_dtype, - validity, - )) + let data = DecimalData::new_handle(values, values_type, decimal_dtype, validity); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Decimal, dtype, len, data)) .vortex_expect("DecimalData is always valid") } @@ -526,9 +540,12 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> Self { - Array::try_from_data(unsafe { + let data = unsafe { DecimalData::new_unchecked_handle(values, values_type, decimal_dtype, validity) - }) + }; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Decimal, dtype, len, data)) .vortex_expect("DecimalData is always valid") } } diff --git a/vortex-array/src/arrays/decimal/vtable/mod.rs b/vortex-array/src/arrays/decimal/vtable/mod.rs index 922b1b80fe2..8c2b8a9e65c 100644 --- a/vortex-array/src/arrays/decimal/vtable/mod.rs +++ b/vortex-array/src/arrays/decimal/vtable/mod.rs @@ -53,38 +53,13 @@ pub struct DecimalMetadata { impl VTable for Decimal { type ArrayData = DecimalData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Decimal - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &DecimalData) -> usize { - let divisor = match array.values_type { - DecimalType::I8 => 1, - DecimalType::I16 => 2, - DecimalType::I32 => 4, - DecimalType::I64 => 8, - DecimalType::I128 => 16, - DecimalType::I256 => 32, - }; - array.values.len() / divisor - } - - fn dtype(array: &DecimalData) -> &DType { - &array.dtype - } - - fn stats(array: &DecimalData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &DecimalData, state: &mut H, precision: Precision) { array.values.array_hash(state, precision); std::mem::discriminant(&array.values_type).hash(state); @@ -115,34 +90,23 @@ impl VTable for Decimal { } } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(DecimalMetadata { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some(ProstMetadata(DecimalMetadata { values_type: array.values_type() as i32, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + }) + .serialize())) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - let metadata = ProstMetadata::::deserialize(bytes)?; - Ok(ProstMetadata(metadata)) - } - - fn build( + &self, dtype: &DType, - len: usize, - metadata: &Self::Metadata, + len: usize, metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = ProstMetadata::::deserialize(metadata)?; if buffers.len() != 1 { vortex_bail!("Expected 1 buffer, got {}", buffers.len()); } diff --git a/vortex-array/src/arrays/dict/array.rs b/vortex-array/src/arrays/dict/array.rs index ceff69fa242..caad6c60b30 100644 --- a/vortex-array/src/arrays/dict/array.rs +++ b/vortex-array/src/arrays/dict/array.rs @@ -11,6 +11,7 @@ use vortex_mask::AllOr; use crate::ArrayRef; use crate::ToCanonical; use crate::array::Array; +use crate::array::ArrayNew; use crate::arrays::Dict; use crate::dtype::DType; use crate::dtype::PType; @@ -204,12 +205,19 @@ impl DictData { impl Array { /// Build a new `DictArray` from its components, `codes` and `values`. pub fn new(codes: ArrayRef, values: ArrayRef) -> Self { - Array::try_from_data(DictData::new(codes, values)).vortex_expect("DictData is always valid") + let data = DictData::new(codes, values); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Dict, dtype, len, data)) + .vortex_expect("DictData is always valid") } /// Build a new `DictArray` from its components, `codes` and `values`. pub fn try_new(codes: ArrayRef, values: ArrayRef) -> VortexResult { - Array::try_from_data(DictData::try_new(codes, values)?) + let data = DictData::try_new(codes, values)?; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Dict, dtype, len, data)) } /// Build a new `DictArray` without validating the codes or values. @@ -218,7 +226,10 @@ impl Array { /// /// See [`DictData::new_unchecked`]. pub unsafe fn new_unchecked(codes: ArrayRef, values: ArrayRef) -> Self { - Array::try_from_data(unsafe { DictData::new_unchecked(codes, values) }) + let data = unsafe { DictData::new_unchecked(codes, values) }; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Dict, dtype, len, data)) .vortex_expect("DictData is always valid") } @@ -228,10 +239,13 @@ impl Array { /// /// See [`DictData::set_all_values_referenced`]. pub unsafe fn set_all_values_referenced(self, all_values_referenced: bool) -> Self { - Array::try_from_data(unsafe { + let dtype = self.dtype().clone(); + let len = self.len(); + let data = unsafe { self.into_data() .set_all_values_referenced(all_values_referenced) - }) + }; + Array::try_from_parts(ArrayNew::new(Dict, dtype, len, data)) .vortex_expect("data is always valid") } } diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index f65e2d43412..bf5a1f622a0 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -28,7 +28,7 @@ use crate::array::ArrayId; use crate::array::ArrayView; use crate::array::VTable; use crate::arrays::Primitive; -use crate::arrays::constant::ConstantData; +use crate::arrays::ConstantArray; use crate::arrays::dict::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -59,30 +59,13 @@ impl Dict { impl VTable for Dict { type ArrayData = DictData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Dict - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &DictData) -> usize { - array.codes().len() - } - - fn dtype(array: &DictData) -> &DType { - &array.dtype - } - - fn stats(array: &DictData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &DictData, state: &mut H, precision: Precision) { array.codes().array_hash(state, precision); array.values().array_hash(state, precision); @@ -105,8 +88,8 @@ impl VTable for Dict { None } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(DictMetadata { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some(ProstMetadata(DictMetadata { codes_ptype: PType::try_from(array.codes().dtype())? as i32, values_len: u32::try_from(array.values().len()).map_err(|_| { vortex_err!( @@ -116,31 +99,20 @@ impl VTable for Dict { })?, is_nullable_codes: Some(array.codes().dtype().is_nullable()), all_values_referenced: Some(array.all_values_referenced), - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + }) + .serialize())) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - let metadata = ::deserialize(bytes)?; - Ok(ProstMetadata(metadata)) - } - - fn build( + &self, dtype: &DType, - len: usize, - metadata: &Self::Metadata, + len: usize, metadata: &[u8], + _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = ProstMetadata::::deserialize(metadata)?; if children.len() != 2 { vortex_bail!( "Expected 2 children for dict encoding, found {}", @@ -197,7 +169,7 @@ impl VTable for Dict { // Also not the check to do here it take value validity using code validity, but this approx // is correct. if array.codes().all_invalid()? { - return Ok(ExecutionResult::done(ConstantData::new( + return Ok(ExecutionResult::done(ConstantArray::new( Scalar::null(array.dtype().as_nullable()), array.codes().len(), ))); diff --git a/vortex-array/src/arrays/dict/vtable/validity.rs b/vortex-array/src/arrays/dict/vtable/validity.rs index cdaeb4a08d5..708d05c4ce3 100644 --- a/vortex-array/src/arrays/dict/vtable/validity.rs +++ b/vortex-array/src/arrays/dict/vtable/validity.rs @@ -7,6 +7,7 @@ use super::Dict; use crate::IntoArray; use crate::array::ArrayView; use crate::array::ValidityVTable; +use crate::arrays::DictArray; use crate::arrays::dict::DictData; use crate::builtins::ArrayBuiltins; use crate::dtype::Nullability; @@ -31,15 +32,15 @@ impl ValidityVTable for Dict { (Validity::AllValid | Validity::NonNullable, Validity::Array(values_validity)) => { // We know codes are all valid, so the cast is free. let codes = array.codes().cast(array.codes().dtype().as_nonnullable())?; - Validity::Array( - unsafe { DictData::new_unchecked(codes, values_validity) }.into_array(), - ) + Validity::Array(unsafe { DictArray::new_unchecked(codes, values_validity) } + .into_array()) } (Validity::Array(_codes_validity), Validity::Array(values_validity)) => { // Create a mask representing "is the value at codes[i] valid?" - let values_valid_mask = - unsafe { DictData::new_unchecked(array.codes().clone(), values_validity) } - .into_array(); + let values_valid_mask = unsafe { + DictArray::new_unchecked(array.codes().clone(), values_validity) + } + .into_array(); let values_valid_mask = values_valid_mask .fill_null(Scalar::bool(false, Nullability::NonNullable))?; diff --git a/vortex-array/src/arrays/extension/array.rs b/vortex-array/src/arrays/extension/array.rs index f641b1202d4..146ab9540b3 100644 --- a/vortex-array/src/arrays/extension/array.rs +++ b/vortex-array/src/arrays/extension/array.rs @@ -6,6 +6,7 @@ use vortex_error::VortexResult; use crate::ArrayRef; use crate::array::Array; +use crate::array::ArrayNew; use crate::arrays::Extension; use crate::dtype::DType; use crate::dtype::extension::ExtDTypeRef; @@ -154,12 +155,18 @@ impl Array { /// /// Panics if the storage array is not compatible with the extension dtype. pub fn new(ext_dtype: ExtDTypeRef, storage_array: ArrayRef) -> Self { - Array::try_from_data(ExtensionData::new(ext_dtype, storage_array)) + let dtype = DType::Extension(ext_dtype.clone()); + let len = storage_array.len(); + let data = ExtensionData::new(ext_dtype, storage_array); + Array::try_from_parts(ArrayNew::new(Extension, dtype, len, data)) .vortex_expect("ExtensionData is always valid") } /// Tries to construct a new `ExtensionArray`. pub fn try_new(ext_dtype: ExtDTypeRef, storage_array: ArrayRef) -> VortexResult { - Array::try_from_data(ExtensionData::try_new(ext_dtype, storage_array)?) + let dtype = DType::Extension(ext_dtype.clone()); + let len = storage_array.len(); + let data = ExtensionData::try_new(ext_dtype, storage_array)?; + Array::try_from_parts(ArrayNew::new(Extension, dtype, len, data)) } } diff --git a/vortex-array/src/arrays/extension/vtable/mod.rs b/vortex-array/src/arrays/extension/vtable/mod.rs index 7eb48c10228..e61d40fd2c9 100644 --- a/vortex-array/src/arrays/extension/vtable/mod.rs +++ b/vortex-array/src/arrays/extension/vtable/mod.rs @@ -39,30 +39,13 @@ vtable!(Extension, Extension, ExtensionData); impl VTable for Extension { type ArrayData = ExtensionData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Extension - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ExtensionData) -> usize { - array.storage_array().len() - } - - fn dtype(array: &ExtensionData) -> &DType { - &array.dtype - } - - fn stats(array: &ExtensionData) -> &ArrayStats { - &array.stats_set - } - fn array_hash( array: &ExtensionData, state: &mut H, @@ -97,31 +80,20 @@ impl VTable for Extension { SLOT_NAMES[idx].to_string() } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) - } - - fn build( + &self, dtype: &DType, - len: usize, - _metadata: &Self::Metadata, + len: usize, metadata: &[u8], + _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + ::deserialize(metadata)?; let DType::Extension(ext_dtype) = dtype else { vortex_bail!("Not an extension DType"); }; diff --git a/vortex-array/src/arrays/filter/array.rs b/vortex-array/src/arrays/filter/array.rs index 04f4312ca1f..ecae1086948 100644 --- a/vortex-array/src/arrays/filter/array.rs +++ b/vortex-array/src/arrays/filter/array.rs @@ -8,6 +8,7 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::array::Array; +use crate::array::ArrayNew; use crate::arrays::Filter; use crate::dtype::DType; use crate::stats::ArrayStats; @@ -93,13 +94,19 @@ impl FilterData { impl Array { /// Creates a new `FilterArray`. pub fn new(array: ArrayRef, mask: Mask) -> Self { - Array::try_from_data(FilterData::new(array, mask)) + let dtype = array.dtype().clone(); + let len = mask.true_count(); + let data = FilterData::new(array, mask); + Array::try_from_parts(ArrayNew::new(Filter, dtype, len, data)) .vortex_expect("FilterData is always valid") } /// Constructs a new `FilterArray`. pub fn try_new(array: ArrayRef, mask: Mask) -> VortexResult { - Array::try_from_data(FilterData::try_new(array, mask)?) + let dtype = array.dtype().clone(); + let len = mask.true_count(); + let data = FilterData::try_new(array, mask)?; + Array::try_from_parts(ArrayNew::new(Filter, dtype, len, data)) } } diff --git a/vortex-array/src/arrays/filter/vtable.rs b/vortex-array/src/arrays/filter/vtable.rs index 42448d7ddd9..e174c298f58 100644 --- a/vortex-array/src/arrays/filter/vtable.rs +++ b/vortex-array/src/arrays/filter/vtable.rs @@ -51,29 +51,13 @@ impl Filter { impl VTable for Filter { type ArrayData = FilterData; - type Metadata = FilterMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &FilterData) -> &Self { - &Filter - } fn id(&self) -> ArrayId { Self::ID } - fn len(array: &FilterData) -> usize { - array.mask.true_count() - } - - fn dtype(array: &FilterData) -> &DType { - array.child().dtype() - } - - fn stats(array: &FilterData) -> &ArrayStats { - &array.stats - } - fn array_hash(array: &FilterData, state: &mut H, precision: Precision) { array.child().array_hash(state, precision); array.mask.array_hash(state, precision); @@ -104,35 +88,21 @@ impl VTable for Filter { SLOT_NAMES[idx].to_string() } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(FilterMetadata(array.mask.clone())) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { // TODO(joe): make this configurable vortex_bail!("Filter array is not serializable") } fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - vortex_bail!("Filter array is not serializable") - } - - fn build( + &self, dtype: &DType, - len: usize, - metadata: &FilterMetadata, + len: usize, _metadata: &[u8], + _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { - assert_eq!(len, metadata.0.true_count()); - let child = children.get(0, dtype, metadata.0.len())?; - FilterData::try_new(child, metadata.0.clone()) + vortex_bail!("Filter array is not serializable") } fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { diff --git a/vortex-array/src/arrays/fixed_size_list/array.rs b/vortex-array/src/arrays/fixed_size_list/array.rs index 9d5dda32f1b..04618e28791 100644 --- a/vortex-array/src/arrays/fixed_size_list/array.rs +++ b/vortex-array/src/arrays/fixed_size_list/array.rs @@ -9,6 +9,7 @@ use vortex_error::vortex_ensure; use crate::ArrayRef; use crate::array::Array; +use crate::array::ArrayNew; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::FixedSizeList; @@ -265,7 +266,10 @@ impl FixedSizeListData { impl Array { /// Creates a new `FixedSizeListArray`. pub fn new(elements: ArrayRef, list_size: u32, validity: Validity, len: usize) -> Self { - Array::try_from_data(FixedSizeListData::new(elements, list_size, validity, len)) + let data = FixedSizeListData::new(elements, list_size, validity, len); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(FixedSizeList, dtype, len, data)) .vortex_expect("FixedSizeListData is always valid") } @@ -276,9 +280,10 @@ impl Array { validity: Validity, len: usize, ) -> VortexResult { - Array::try_from_data(FixedSizeListData::try_new( - elements, list_size, validity, len, - )?) + let data = FixedSizeListData::try_new(elements, list_size, validity, len)?; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(FixedSizeList, dtype, len, data)) } /// Creates a new `FixedSizeListArray` without validation. @@ -292,9 +297,10 @@ impl Array { validity: Validity, len: usize, ) -> Self { - Array::try_from_data(unsafe { - FixedSizeListData::new_unchecked(elements, list_size, validity, len) - }) + let data = unsafe { FixedSizeListData::new_unchecked(elements, list_size, validity, len) }; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(FixedSizeList, dtype, len, data)) .vortex_expect("FixedSizeListData is always valid") } } diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs index 31fa5342185..5f5b6d12b51 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs @@ -46,29 +46,13 @@ impl FixedSizeList { impl VTable for FixedSizeList { type ArrayData = FixedSizeListData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &FixedSizeListData) -> &Self { - &FixedSizeList - } fn id(&self) -> ArrayId { Self::ID } - fn len(array: &FixedSizeListData) -> usize { - array.len - } - - fn dtype(array: &FixedSizeListData) -> &DType { - &array.dtype - } - - fn stats(array: &FixedSizeListData) -> &ArrayStats { - &array.stats_set - } - fn array_hash( array: &FixedSizeListData, state: &mut H, @@ -118,34 +102,20 @@ impl VTable for FixedSizeList { Self::PARENT_KERNELS.execute(array, parent, child_idx, ctx) } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) - } - - /// Builds a [`FixedSizeListArray`]. - /// - /// This method expects 1 or 2 children (a second child indicates a validity array). - fn build( + &self, dtype: &DType, - len: usize, - _metadata: &Self::Metadata, + len: usize, metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + ::deserialize(metadata)?; vortex_ensure!( buffers.is_empty(), "`FixedSizeList::build` expects no buffers" diff --git a/vortex-array/src/arrays/list/array.rs b/vortex-array/src/arrays/list/array.rs index 2678ab551e2..5cdf75f7945 100644 --- a/vortex-array/src/arrays/list/array.rs +++ b/vortex-array/src/arrays/list/array.rs @@ -16,6 +16,7 @@ use crate::LEGACY_SESSION; use crate::VortexSessionExecute; use crate::aggregate_fn::fns::min_max::min_max; use crate::array::Array; +use crate::array::ArrayNew; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::ConstantArray; @@ -363,7 +364,10 @@ impl ListData { impl Array { /// Creates a new `ListArray`. pub fn new(elements: ArrayRef, offsets: ArrayRef, validity: Validity) -> Self { - Array::try_from_data(ListData::new(elements, offsets, validity)) + let data = ListData::new(elements, offsets, validity); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(List, dtype, len, data)) .vortex_expect("ListData is always valid") } @@ -373,7 +377,10 @@ impl Array { offsets: ArrayRef, validity: Validity, ) -> VortexResult { - Array::try_from_data(ListData::try_new(elements, offsets, validity)?) + let data = ListData::try_new(elements, offsets, validity)?; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(List, dtype, len, data)) } /// Creates a new `ListArray` without validation. @@ -382,7 +389,10 @@ impl Array { /// /// See [`ListData::new_unchecked`]. pub unsafe fn new_unchecked(elements: ArrayRef, offsets: ArrayRef, validity: Validity) -> Self { - Array::try_from_data(unsafe { ListData::new_unchecked(elements, offsets, validity) }) + let data = unsafe { ListData::new_unchecked(elements, offsets, validity) }; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(List, dtype, len, data)) .vortex_expect("ListData is always valid") } } @@ -393,7 +403,10 @@ impl ListData { if recurse && elements.is_canonical() { elements = elements.to_canonical()?.compact()?.into_array(); } else if recurse && let Some(child_list_array) = elements.as_opt::() { - elements = child_list_array.reset_offsets(recurse)?.into_array(); + let data = child_list_array.reset_offsets(recurse)?; + let dtype = data.dtype().clone(); + let len = data.len(); + elements = Array::try_from_parts(ArrayNew::new(List, dtype, len, data))?.into_array(); } let offsets = self.offsets(); diff --git a/vortex-array/src/arrays/list/vtable/mod.rs b/vortex-array/src/arrays/list/vtable/mod.rs index 051ee80d2d0..65bc4b92fe9 100644 --- a/vortex-array/src/arrays/list/vtable/mod.rs +++ b/vortex-array/src/arrays/list/vtable/mod.rs @@ -50,29 +50,13 @@ pub struct ListMetadata { impl VTable for List { type ArrayData = ListData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &ListData) -> &Self { - &List - } fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ListData) -> usize { - array.offsets().len().saturating_sub(1) - } - - fn dtype(array: &ListData) -> &DType { - &array.dtype - } - - fn stats(array: &ListData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &ListData, state: &mut H, precision: Precision) { array.elements().array_hash(state, precision); array.offsets().array_hash(state, precision); @@ -105,36 +89,24 @@ impl VTable for List { PARENT_RULES.evaluate(array, parent, child_idx) } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(ListMetadata { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some(ProstMetadata(ListMetadata { elements_len: array.elements().len() as u64, offset_ptype: PType::try_from(array.offsets().dtype())? as i32, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(SerializeMetadata::serialize(metadata))) + }) + .serialize())) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata( - as DeserializeMetadata>::deserialize(bytes)?, - )) - } - - fn build( + &self, dtype: &DType, - len: usize, - metadata: &Self::Metadata, + len: usize, metadata: &[u8], + _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = ProstMetadata::::deserialize(metadata)?; let validity = if children.len() == 2 { Validity::from(dtype.nullability()) } else if children.len() == 3 { @@ -150,12 +122,12 @@ impl VTable for List { let elements = children.get( 0, element_dtype.as_ref(), - usize::try_from(metadata.0.elements_len)?, + usize::try_from(metadata.elements_len)?, )?; let offsets = children.get( 1, - &DType::Primitive(metadata.0.offset_ptype(), Nullability::NonNullable), + &DType::Primitive(metadata.offset_ptype(), Nullability::NonNullable), len + 1, )?; diff --git a/vortex-array/src/arrays/listview/array.rs b/vortex-array/src/arrays/listview/array.rs index 7a8c7122d4c..a2881f374c1 100644 --- a/vortex-array/src/arrays/listview/array.rs +++ b/vortex-array/src/arrays/listview/array.rs @@ -13,6 +13,7 @@ use vortex_error::vortex_err; use crate::ArrayRef; use crate::ToCanonical; use crate::array::Array; +use crate::array::ArrayNew; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::ListView; @@ -485,7 +486,10 @@ impl ListViewData { impl Array { /// Creates a new `ListViewArray`. pub fn new(elements: ArrayRef, offsets: ArrayRef, sizes: ArrayRef, validity: Validity) -> Self { - Array::try_from_data(ListViewData::new(elements, offsets, sizes, validity)) + let data = ListViewData::new(elements, offsets, sizes, validity); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(ListView, dtype, len, data)) .vortex_expect("ListViewData is always valid") } @@ -496,7 +500,10 @@ impl Array { sizes: ArrayRef, validity: Validity, ) -> VortexResult { - Array::try_from_data(ListViewData::try_new(elements, offsets, sizes, validity)?) + let data = ListViewData::try_new(elements, offsets, sizes, validity)?; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(ListView, dtype, len, data)) } /// Creates a new `ListViewArray` without validation. @@ -510,9 +517,10 @@ impl Array { sizes: ArrayRef, validity: Validity, ) -> Self { - Array::try_from_data(unsafe { - ListViewData::new_unchecked(elements, offsets, sizes, validity) - }) + let data = unsafe { ListViewData::new_unchecked(elements, offsets, sizes, validity) }; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(ListView, dtype, len, data)) .vortex_expect("ListViewData is always valid") } @@ -522,7 +530,10 @@ impl Array { /// /// See [`ListViewData::with_zero_copy_to_list`]. pub unsafe fn with_zero_copy_to_list(self, is_zctl: bool) -> Self { - Array::try_from_data(unsafe { self.into_data().with_zero_copy_to_list(is_zctl) }) + let dtype = self.dtype().clone(); + let len = self.len(); + let data = unsafe { self.into_data().with_zero_copy_to_list(is_zctl) }; + Array::try_from_parts(ArrayNew::new(ListView, dtype, len, data)) .vortex_expect("data is always valid") } } diff --git a/vortex-array/src/arrays/listview/conversion.rs b/vortex-array/src/arrays/listview/conversion.rs index be4ac64ab41..6eef7440064 100644 --- a/vortex-array/src/arrays/listview/conversion.rs +++ b/vortex-array/src/arrays/listview/conversion.rs @@ -10,8 +10,10 @@ use crate::ExecutionCtx; use crate::IntoArray; use crate::ToCanonical; use crate::array::Array; +use crate::array::ArrayNew; use crate::arrays::ExtensionArray; use crate::arrays::FixedSizeListArray; +use crate::arrays::List; use crate::arrays::ListArray; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; @@ -36,8 +38,10 @@ pub fn list_view_from_list(list: ListArray, ctx: &mut ExecutionCtx) -> VortexRes // We reset the offsets here because mostly for convenience, and also because callers of this // function might not expect the output `ListViewArray` to have a bunch of leading and trailing // garbage data when they turn it back into a `ListArray`. - let list: ListArray = - Array::try_from_data(list.reset_offsets(false).vortex_expect("This can't fail"))?; + let data = list.reset_offsets(false).vortex_expect("This can't fail"); + let dtype = data.dtype().clone(); + let len = data.len(); + let list: ListArray = Array::try_from_parts(ArrayNew::new(List, dtype, len, data))?; let list_offsets = list.offsets().clone(); diff --git a/vortex-array/src/arrays/listview/vtable/mod.rs b/vortex-array/src/arrays/listview/vtable/mod.rs index e8fab2f9d82..2c3ceffe9fa 100644 --- a/vortex-array/src/arrays/listview/vtable/mod.rs +++ b/vortex-array/src/arrays/listview/vtable/mod.rs @@ -56,30 +56,13 @@ pub struct ListViewMetadata { impl VTable for ListView { type ArrayData = ListViewData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &ListViewData) -> &Self { - &ListView - } fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ListViewData) -> usize { - debug_assert_eq!(array.offsets().len(), array.sizes().len()); - array.offsets().len() - } - - fn dtype(array: &ListViewData) -> &DType { - &array.dtype - } - - fn stats(array: &ListViewData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &ListViewData, state: &mut H, precision: Precision) { array.elements().array_hash(state, precision); array.offsets().array_hash(state, precision); @@ -106,36 +89,25 @@ impl VTable for ListView { vortex_panic!("ListViewArray buffer_name index {idx} out of bounds") } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(ListViewMetadata { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some(ProstMetadata(ListViewMetadata { elements_len: array.elements().len() as u64, offset_ptype: PType::try_from(array.offsets().dtype())? as i32, size_ptype: PType::try_from(array.sizes().dtype())? as i32, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + }) + .serialize())) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - let metadata = ::deserialize(bytes)?; - Ok(ProstMetadata(metadata)) - } - - fn build( + &self, dtype: &DType, - len: usize, - metadata: &Self::Metadata, + len: usize, metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = ProstMetadata::::deserialize(metadata)?; vortex_ensure!( buffers.is_empty(), "`ListViewArray::build` expects no buffers" @@ -161,20 +133,20 @@ impl VTable for ListView { let elements = children.get( 0, element_dtype.as_ref(), - usize::try_from(metadata.0.elements_len)?, + usize::try_from(metadata.elements_len)?, )?; // Get offsets with proper type from metadata. let offsets = children.get( 1, - &DType::Primitive(metadata.0.offset_ptype(), Nullability::NonNullable), + &DType::Primitive(metadata.offset_ptype(), Nullability::NonNullable), len, )?; // Get sizes with proper type from metadata. let sizes = children.get( 2, - &DType::Primitive(metadata.0.size_ptype(), Nullability::NonNullable), + &DType::Primitive(metadata.size_ptype(), Nullability::NonNullable), len, )?; diff --git a/vortex-array/src/arrays/masked/array.rs b/vortex-array/src/arrays/masked/array.rs index 0c2bc773817..cdda1d10e44 100644 --- a/vortex-array/src/arrays/masked/array.rs +++ b/vortex-array/src/arrays/masked/array.rs @@ -7,6 +7,7 @@ use vortex_error::vortex_bail; use crate::ArrayRef; use crate::array::Array; +use crate::array::ArrayNew; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Masked; @@ -93,6 +94,9 @@ impl MaskedData { impl Array { /// Constructs a new `MaskedArray`. pub fn try_new(child: ArrayRef, validity: Validity) -> VortexResult { - Array::try_from_data(MaskedData::try_new(child, validity)?) + let dtype = child.dtype().as_nullable(); + let len = child.len(); + let data = MaskedData::try_new(child, validity)?; + Array::try_from_parts(ArrayNew::new(Masked, dtype, len, data)) } } diff --git a/vortex-array/src/arrays/masked/compute/filter.rs b/vortex-array/src/arrays/masked/compute/filter.rs index a96c57feb1f..b810248b614 100644 --- a/vortex-array/src/arrays/masked/compute/filter.rs +++ b/vortex-array/src/arrays/masked/compute/filter.rs @@ -8,8 +8,8 @@ use crate::ArrayRef; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Masked; +use crate::arrays::MaskedArray; use crate::arrays::filter::FilterReduce; -use crate::arrays::masked::MaskedData; impl FilterReduce for Masked { fn filter(array: ArrayView<'_, Masked>, mask: &Mask) -> VortexResult> { @@ -21,9 +21,7 @@ impl FilterReduce for Masked { let filtered_child = array.child().filter(mask.clone())?; // Construct new MaskedArray - Ok(Some( - MaskedData::try_new(filtered_child, filtered_validity)?.into_array(), - )) + Ok(Some(MaskedArray::try_new(filtered_child, filtered_validity)?.into_array())) } } diff --git a/vortex-array/src/arrays/masked/compute/slice.rs b/vortex-array/src/arrays/masked/compute/slice.rs index afa9b73a6e6..1be55ea6c30 100644 --- a/vortex-array/src/arrays/masked/compute/slice.rs +++ b/vortex-array/src/arrays/masked/compute/slice.rs @@ -9,7 +9,7 @@ use crate::ArrayRef; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Masked; -use crate::arrays::masked::MaskedData; +use crate::arrays::MaskedArray; use crate::arrays::slice::SliceReduce; impl SliceReduce for Masked { @@ -17,6 +17,6 @@ impl SliceReduce for Masked { let child = array.child().slice(range.clone())?; let validity = array.validity().slice(range)?; - Ok(Some(MaskedData::try_new(child, validity)?.into_array())) + Ok(Some(MaskedArray::try_new(child, validity)?.into_array())) } } diff --git a/vortex-array/src/arrays/masked/compute/take.rs b/vortex-array/src/arrays/masked/compute/take.rs index bd9a45b4711..cfe1653ea4c 100644 --- a/vortex-array/src/arrays/masked/compute/take.rs +++ b/vortex-array/src/arrays/masked/compute/take.rs @@ -7,8 +7,8 @@ use crate::ArrayRef; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Masked; +use crate::arrays::MaskedArray; use crate::arrays::dict::TakeReduce; -use crate::arrays::masked::MaskedData; use crate::builtins::ArrayBuiltins; use crate::scalar::Scalar; @@ -27,9 +27,7 @@ impl TakeReduce for Masked { let taken_validity = array.validity().take(indices)?; // Construct new MaskedArray - Ok(Some( - MaskedData::try_new(taken_child, taken_validity)?.into_array(), - )) + Ok(Some(MaskedArray::try_new(taken_child, taken_validity)?.into_array())) } } diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index b7069998f99..809a301d534 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -19,7 +19,7 @@ use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; use crate::array::VTable; -use crate::arrays::constant::ConstantData; +use crate::arrays::ConstantArray; use crate::arrays::masked::MaskedData; use crate::arrays::masked::array::NUM_SLOTS; use crate::arrays::masked::array::SLOT_NAMES; @@ -48,30 +48,13 @@ impl Masked { impl VTable for Masked { type ArrayData = MaskedData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Masked - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &MaskedData) -> usize { - array.child().len() - } - - fn dtype(array: &MaskedData) -> &DType { - &array.dtype - } - - fn stats(array: &MaskedData) -> &ArrayStats { - &array.stats - } - fn array_hash(array: &MaskedData, state: &mut H, precision: Precision) { array.child().array_hash(state, precision); array.validity().array_hash(state, precision); @@ -94,31 +77,20 @@ impl VTable for Masked { None } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) - } - - fn build( + &self, dtype: &DType, - len: usize, - _metadata: &Self::Metadata, + len: usize, metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + ::deserialize(metadata)?; if !buffers.is_empty() { vortex_bail!("Expected 0 buffer, got {}", buffers.len()); } @@ -147,7 +119,7 @@ impl VTable for Masked { // Fast path: all masked means result is all nulls. if validity_mask.all_false() { return Ok(ExecutionResult::done( - ConstantData::new(Scalar::null(array.dtype().as_nullable()), array.len()) + ConstantArray::new(Scalar::null(array.dtype().as_nullable()), array.len()) .into_array(), )); } diff --git a/vortex-array/src/arrays/null/mod.rs b/vortex-array/src/arrays/null/mod.rs index 2643a9ae4cf..b240a7dc2d0 100644 --- a/vortex-array/src/arrays/null/mod.rs +++ b/vortex-array/src/arrays/null/mod.rs @@ -14,6 +14,7 @@ use crate::ExecutionResult; use crate::Precision; use crate::array::Array; use crate::array::ArrayId; +use crate::array::ArrayNew; use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::array::VTable; @@ -36,30 +37,13 @@ vtable!(Null, Null, NullData); impl VTable for Null { type ArrayData = NullData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Null - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &NullData) -> usize { - array.len - } - - fn dtype(_array: &NullData) -> &DType { - &DType::Null - } - - fn stats(array: &NullData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(_array: &NullData, _state: &mut H, _precision: Precision) { // len and dtype are hashed by ArrayInner; NullData has no additional fields. } @@ -99,31 +83,20 @@ impl VTable for Null { Ok(()) } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], + &self, _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) - } + len: usize, metadata: &[u8], - fn build( - _dtype: &DType, - len: usize, - _metadata: &Self::Metadata, _buffers: &[BufferHandle], _children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + ::deserialize(metadata)?; Ok(NullData::new(len)) } @@ -183,7 +156,8 @@ impl Null { impl Array { pub fn new(len: usize) -> Self { - Array::try_from_data(NullData::new(len)).vortex_expect("NullData is always valid") + Array::try_from_parts(ArrayNew::new(Null, DType::Null, len, NullData::new(len))) + .vortex_expect("NullData is always valid") } } diff --git a/vortex-array/src/arrays/patched/array.rs b/vortex-array/src/arrays/patched/array.rs index 9c0d541f5db..4366c2fae2e 100644 --- a/vortex-array/src/arrays/patched/array.rs +++ b/vortex-array/src/arrays/patched/array.rs @@ -14,7 +14,10 @@ use crate::ArrayRef; use crate::Canonical; use crate::ExecutionCtx; use crate::IntoArray; +use crate::array::Array; +use crate::array::ArrayNew; use crate::arrays::PrimitiveArray; +use crate::arrays::Patched; use crate::arrays::patched::TransposedPatches; use crate::arrays::patched::patch_lanes; use crate::buffer::BufferHandle; @@ -130,6 +133,23 @@ pub struct PatchedArray { pub(super) stats_set: ArrayStats, } +impl IntoArray for PatchedArray { + fn into_array(self) -> ArrayRef { + let dtype = self.base_array().dtype().clone(); + let len = self.len; + let stats = self.stats_set.clone(); + Array::::try_from_parts(ArrayNew::new(Patched, dtype, len, self).with_stats(stats)) + .vortex_expect("PatchedArray is always valid") + .into_array() + } +} + +impl From for ArrayRef { + fn from(value: PatchedArray) -> Self { + value.into_array() + } +} + impl PatchedArray { /// Create a new `PatchedArray` from a child array and a set of [`Patches`]. /// diff --git a/vortex-array/src/arrays/patched/vtable/mod.rs b/vortex-array/src/arrays/patched/vtable/mod.rs index eb5c887772a..671afe60291 100644 --- a/vortex-array/src/arrays/patched/vtable/mod.rs +++ b/vortex-array/src/arrays/patched/vtable/mod.rs @@ -80,30 +80,13 @@ pub struct PatchedMetadata { impl VTable for Patched { type ArrayData = PatchedArray; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Patched - } - fn id(&self) -> ArrayId { ArrayId::new_ref("vortex.patched") } - fn len(array: &Self::ArrayData) -> usize { - array.len - } - - fn dtype(array: &Self::ArrayData) -> &DType { - array.base_array().dtype() - } - - fn stats(array: &Self::ArrayData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &PatchedArray, state: &mut H, precision: Precision) { array.offset.hash(state); array.n_lanes.hash(state); @@ -150,27 +133,45 @@ impl VTable for Patched { } } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(PatchedMetadata { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some(ProstMetadata(PatchedMetadata { n_patches: u32::try_from(array.patch_indices().len())?, n_lanes: u32::try_from(array.n_lanes)?, offset: u32::try_from(array.offset)?, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + }) + .serialize())) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, + &self, + dtype: &DType, + len: usize, + metadata: &[u8], _buffers: &[BufferHandle], + children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { - let inner = as DeserializeMetadata>::deserialize(bytes)?; - Ok(ProstMetadata(inner)) + ) -> VortexResult { + let metadata = ProstMetadata::::deserialize(metadata)?; + let n_patches = metadata.n_patches as usize; + let n_lanes = metadata.n_lanes as usize; + let offset = metadata.offset as usize; + + // n_chunks should correspond to the chunk in the `inner`. + // After slicing when offset > 0, there may be additional chunks. + let n_chunks = (len + offset).div_ceil(1024); + + let inner = children.get(0, dtype, len)?; + let lane_offsets = children.get(1, PType::U32.into(), n_chunks * n_lanes + 1)?; + let indices = children.get(2, PType::U16.into(), n_patches)?; + let values = children.get(3, dtype, n_patches)?; + + Ok(PatchedArray { + slots: vec![Some(inner), Some(lane_offsets), Some(indices), Some(values)], + n_lanes, + offset, + len, + stats_set: ArrayStats::default(), + }) } fn append_to_builder( @@ -236,35 +237,6 @@ impl VTable for Patched { Ok(()) } - fn build( - dtype: &DType, - len: usize, - metadata: &Self::Metadata, - _buffers: &[BufferHandle], - children: &dyn ArrayChildren, - ) -> VortexResult { - let n_patches = metadata.n_patches as usize; - let n_lanes = metadata.n_lanes as usize; - let offset = metadata.offset as usize; - - // n_chunks should correspond to the chunk in the `inner`. - // After slicing when offset > 0, there may be additional chunks. - let n_chunks = (len + offset).div_ceil(1024); - - let inner = children.get(0, dtype, len)?; - let lane_offsets = children.get(1, PType::U32.into(), n_chunks * n_lanes + 1)?; - let indices = children.get(2, PType::U16.into(), n_patches)?; - let values = children.get(3, dtype, n_patches)?; - - Ok(PatchedArray { - slots: vec![Some(inner), Some(lane_offsets), Some(indices), Some(values)], - n_lanes, - offset, - len, - stats_set: ArrayStats::default(), - }) - } - fn slots(array: ArrayView<'_, Self>) -> &[Option] { &array.data().slots } diff --git a/vortex-array/src/arrays/primitive/array/mod.rs b/vortex-array/src/arrays/primitive/array/mod.rs index 1a7a9a0efdd..b6a592ae6ca 100644 --- a/vortex-array/src/arrays/primitive/array/mod.rs +++ b/vortex-array/src/arrays/primitive/array/mod.rs @@ -15,6 +15,7 @@ use vortex_error::vortex_panic; use crate::ToCanonical; use crate::array::Array; +use crate::array::ArrayNew; use crate::arrays::Primitive; use crate::dtype::DType; use crate::dtype::NativePType; @@ -197,7 +198,10 @@ impl PrimitiveData { impl Array { pub fn empty(nullability: Nullability) -> Self { - Array::try_from_data(PrimitiveData::empty::(nullability)) + let data = PrimitiveData::empty::(nullability); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Primitive, dtype, len, data)) .vortex_expect("PrimitiveData is always valid") } @@ -207,13 +211,19 @@ impl Array { /// /// Panics if the provided components do not satisfy the invariants. pub fn new(buffer: impl Into>, validity: Validity) -> Self { - Array::try_from_data(PrimitiveData::new(buffer, validity)) + let data = PrimitiveData::new(buffer, validity); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Primitive, dtype, len, data)) .vortex_expect("PrimitiveData is always valid") } /// Constructs a new `PrimitiveArray`. pub fn try_new(buffer: Buffer, validity: Validity) -> VortexResult { - Array::try_from_data(PrimitiveData::try_new(buffer, validity)?) + let data = PrimitiveData::try_new(buffer, validity)?; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Primitive, dtype, len, data)) } /// Creates a new `PrimitiveArray` without validation. @@ -222,7 +232,10 @@ impl Array { /// /// See [`PrimitiveData::new_unchecked`]. pub unsafe fn new_unchecked(buffer: Buffer, validity: Validity) -> Self { - Array::try_from_data(unsafe { PrimitiveData::new_unchecked(buffer, validity) }) + let data = unsafe { PrimitiveData::new_unchecked(buffer, validity) }; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Primitive, dtype, len, data)) .vortex_expect("PrimitiveData is always valid") } @@ -236,21 +249,28 @@ impl Array { ptype: PType, validity: Validity, ) -> Self { - Array::try_from_data(unsafe { - PrimitiveData::new_unchecked_from_handle(handle, ptype, validity) - }) + let data = unsafe { PrimitiveData::new_unchecked_from_handle(handle, ptype, validity) }; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Primitive, dtype, len, data)) .vortex_expect("PrimitiveData is always valid") } /// Creates a new `PrimitiveArray` from a [`BufferHandle`]. pub fn from_buffer_handle(handle: BufferHandle, ptype: PType, validity: Validity) -> Self { - Array::try_from_data(PrimitiveData::from_buffer_handle(handle, ptype, validity)) + let data = PrimitiveData::from_buffer_handle(handle, ptype, validity); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Primitive, dtype, len, data)) .vortex_expect("PrimitiveData is always valid") } /// Creates a new `PrimitiveArray` from a [`ByteBuffer`]. pub fn from_byte_buffer(buffer: ByteBuffer, ptype: PType, validity: Validity) -> Self { - Array::try_from_data(PrimitiveData::from_byte_buffer(buffer, ptype, validity)) + let data = PrimitiveData::from_byte_buffer(buffer, ptype, validity); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Primitive, dtype, len, data)) .vortex_expect("PrimitiveData is always valid") } @@ -261,12 +281,15 @@ impl Array { validity: Validity, n_rows: usize, ) -> Self { - Array::try_from_data(PrimitiveData::from_values_byte_buffer( + let data = PrimitiveData::from_values_byte_buffer( valid_elems_buffer, ptype, validity, n_rows, - )) + ); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Primitive, dtype, len, data)) .vortex_expect("PrimitiveData is always valid") } diff --git a/vortex-array/src/arrays/primitive/vtable/mod.rs b/vortex-array/src/arrays/primitive/vtable/mod.rs index db8bd8c5258..97661e8d04c 100644 --- a/vortex-array/src/arrays/primitive/vtable/mod.rs +++ b/vortex-array/src/arrays/primitive/vtable/mod.rs @@ -44,30 +44,13 @@ vtable!(Primitive, Primitive, PrimitiveData); impl VTable for Primitive { type ArrayData = PrimitiveData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Primitive - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &PrimitiveData) -> usize { - array.buffer_handle().len() / array.ptype().byte_width() - } - - fn dtype(array: &PrimitiveData) -> &DType { - &array.dtype - } - - fn stats(array: &PrimitiveData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &PrimitiveData, state: &mut H, precision: Precision) { array.buffer.array_hash(state, precision); array.validity().array_hash(state, precision); @@ -96,31 +79,20 @@ impl VTable for Primitive { } } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) - } - - fn build( + &self, dtype: &DType, - len: usize, - _metadata: &Self::Metadata, + len: usize, metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + ::deserialize(metadata)?; if buffers.len() != 1 { vortex_bail!("Expected 1 buffer, got {}", buffers.len()); } diff --git a/vortex-array/src/arrays/scalar_fn/array.rs b/vortex-array/src/arrays/scalar_fn/array.rs index 029ff0802fa..1227f779603 100644 --- a/vortex-array/src/arrays/scalar_fn/array.rs +++ b/vortex-array/src/arrays/scalar_fn/array.rs @@ -7,6 +7,7 @@ use vortex_error::vortex_ensure; use crate::ArrayRef; use crate::array::Array; +use crate::array::ArrayNew; use crate::arrays::ScalarFnVTable; use crate::dtype::DType; use crate::scalar_fn::ScalarFnRef; @@ -116,6 +117,10 @@ impl Array { children: Vec, len: usize, ) -> VortexResult { - Array::try_from_data(ScalarFnData::try_new(scalar_fn, children, len)?) + let data = ScalarFnData::try_new(scalar_fn, children, len)?; + let dtype = data.dtype().clone(); + let len = data.len(); + let vtable = data.vtable.clone(); + Array::try_from_parts(ArrayNew::new(vtable, dtype, len, data)) } } diff --git a/vortex-array/src/arrays/scalar_fn/rules.rs b/vortex-array/src/arrays/scalar_fn/rules.rs index 40fec2594cf..fc818258240 100644 --- a/vortex-array/src/arrays/scalar_fn/rules.rs +++ b/vortex-array/src/arrays/scalar_fn/rules.rs @@ -19,7 +19,6 @@ use crate::arrays::ScalarFnArray; use crate::arrays::ScalarFnVTable; use crate::arrays::Slice; use crate::arrays::StructArray; -use crate::arrays::scalar_fn::ScalarFnData; use crate::dtype::DType; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ArrayReduceRule; @@ -104,7 +103,7 @@ impl ArrayParentReduceRule for ScalarFnSliceReduceRule { .collect::>()?; Ok(Some( - ScalarFnData::try_new(array.scalar_fn().clone(), children, range.len())?.into_array(), + ScalarFnArray::try_new(array.scalar_fn().clone(), children, range.len())?.into_array(), )) } } diff --git a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs index 5f4d3e2835a..e6a707b8165 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs @@ -23,10 +23,10 @@ use crate::IntoArray; use crate::Precision; use crate::array::Array; use crate::array::ArrayId; +use crate::array::ArrayNew; use crate::array::ArrayView; use crate::array::VTable; use crate::arrays::scalar_fn::array::ScalarFnData; -use crate::arrays::scalar_fn::metadata::ScalarFnMetadata; use crate::arrays::scalar_fn::rules::PARENT_RULES; use crate::arrays::scalar_fn::rules::RULES; use crate::buffer::BufferHandle; @@ -44,7 +44,6 @@ use crate::scalar_fn::ScalarFnRef; use crate::scalar_fn::ScalarFnVTableExt; use crate::scalar_fn::VecExecutionArgs; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::vtable; vtable!(ScalarFn, ScalarFnVTable, ScalarFnData); @@ -56,30 +55,13 @@ pub struct ScalarFnVTable { impl VTable for ScalarFnVTable { type ArrayData = ScalarFnData; - type Metadata = ScalarFnMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(array: &ScalarFnData) -> &Self { - &array.vtable - } - fn id(&self) -> ArrayId { self.scalar_fn.id() } - fn len(array: &ScalarFnData) -> usize { - array.len - } - - fn dtype(array: &ScalarFnData) -> &DType { - &array.dtype - } - - fn stats(array: &ScalarFnData) -> &ArrayStats { - &array.stats - } - fn array_hash(array: &ScalarFnData, state: &mut H, precision: Precision) { array.scalar_fn().hash(state); for child in array.iter_children() { @@ -111,61 +93,21 @@ impl VTable for ScalarFnVTable { None } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - let child_dtypes = array.iter_children().map(|c| c.dtype().clone()).collect(); - Ok(ScalarFnMetadata { - scalar_fn: array.scalar_fn().clone(), - child_dtypes, - }) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { // Not supported Ok(None) } fn deserialize( - _bytes: &[u8], + &self, _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - vortex_bail!("Deserialization of ScalarFnVTable metadata is not supported"); - } + _len: usize, _metadata: &[u8], - fn build( - dtype: &DType, - len: usize, - metadata: &ScalarFnMetadata, _buffers: &[BufferHandle], - children: &dyn ArrayChildren, + _children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { - let children: Vec<_> = metadata - .child_dtypes - .iter() - .enumerate() - .map(|(idx, child_dtype)| children.get(idx, child_dtype, len)) - .try_collect()?; - - #[cfg(debug_assertions)] - { - let child_dtypes: Vec<_> = children.iter().map(|c| c.dtype().clone()).collect(); - vortex_error::vortex_ensure!( - &metadata.scalar_fn.return_dtype(&child_dtypes)? == dtype, - "Return dtype mismatch when building ScalarFnArray" - ); - } - - Ok(ScalarFnData { - vtable: ScalarFnVTable { - scalar_fn: metadata.scalar_fn.clone(), - }, - dtype: dtype.clone(), - len, - slots: children.into_iter().map(Some).collect(), - stats: Default::default(), - }) + vortex_bail!("Deserialization of ScalarFnVTable metadata is not supported"); } fn slots(array: ArrayView<'_, Self>) -> &[Option] { @@ -227,14 +169,17 @@ pub trait ScalarFnArrayExt: scalar_fn::ScalarFnVTable { let child_dtypes = children.iter().map(|c| c.dtype().clone()).collect_vec(); let dtype = scalar_fn.return_dtype(&child_dtypes)?; - Ok(ScalarFnData { + let data = ScalarFnData { vtable: ScalarFnVTable { scalar_fn }, dtype, len, slots: children.into_iter().map(Some).collect(), stats: Default::default(), - } - .into_array()) + }; + let vtable = data.vtable.clone(); + let dtype = data.dtype.clone(); + let len = data.len; + Array::try_from_parts(ArrayNew::new(vtable, dtype, len, data)).map(IntoArray::into_array) } } impl ScalarFnArrayExt for V {} diff --git a/vortex-array/src/arrays/shared/array.rs b/vortex-array/src/arrays/shared/array.rs index c0d4ab33f86..0a8b2e59169 100644 --- a/vortex-array/src/arrays/shared/array.rs +++ b/vortex-array/src/arrays/shared/array.rs @@ -14,6 +14,7 @@ use crate::ArrayRef; use crate::Canonical; use crate::IntoArray; use crate::array::Array; +use crate::array::ArrayNew; use crate::arrays::Shared; use crate::dtype::DType; use crate::stats::ArrayStats; @@ -125,7 +126,10 @@ impl SharedData { impl Array { /// Creates a new `SharedArray`. pub fn new(source: ArrayRef) -> Self { - Array::try_from_data(SharedData::new(source)).vortex_expect("SharedData is always valid") + let dtype = source.dtype().clone(); + let len = source.len(); + Array::try_from_parts(ArrayNew::new(Shared, dtype, len, SharedData::new(source))) + .vortex_expect("SharedData is always valid") } } diff --git a/vortex-array/src/arrays/shared/vtable.rs b/vortex-array/src/arrays/shared/vtable.rs index 8f04b2a37f9..11c4714f87f 100644 --- a/vortex-array/src/arrays/shared/vtable.rs +++ b/vortex-array/src/arrays/shared/vtable.rs @@ -42,29 +42,13 @@ impl Shared { impl VTable for Shared { type ArrayData = SharedData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &SharedData) -> &Self { - &Shared - } fn id(&self) -> ArrayId { Self::ID } - fn len(array: &SharedData) -> usize { - array.current_array_ref().len() - } - - fn dtype(array: &SharedData) -> &DType { - &array.dtype - } - - fn stats(array: &SharedData) -> &ArrayStats { - &array.stats - } - fn array_hash(array: &SharedData, state: &mut H, precision: Precision) { let current = array.current_array_ref(); current.array_hash(state, precision); @@ -110,33 +94,20 @@ impl VTable for Shared { Ok(()) } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { vortex_error::vortex_bail!("Shared array is not serializable") } fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - vortex_error::vortex_bail!("Shared array is not serializable") - } - - fn build( + &self, dtype: &DType, - len: usize, - _metadata: &Self::Metadata, + len: usize, _metadata: &[u8], + _buffers: &[BufferHandle], children: &dyn crate::serde::ArrayChildren, + _session: &VortexSession, ) -> VortexResult { - let child = children.get(0, dtype, len)?; - Ok(SharedData::new(child)) + vortex_error::vortex_bail!("Shared array is not serializable") } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { diff --git a/vortex-array/src/arrays/slice/array.rs b/vortex-array/src/arrays/slice/array.rs index a8ded19335b..1bf3a01399b 100644 --- a/vortex-array/src/arrays/slice/array.rs +++ b/vortex-array/src/arrays/slice/array.rs @@ -9,6 +9,7 @@ use vortex_error::vortex_panic; use crate::ArrayRef; use crate::array::Array; +use crate::array::ArrayNew; use crate::arrays::Slice; use crate::dtype::DType; use crate::stats::ArrayStats; @@ -81,12 +82,18 @@ impl SliceData { impl Array { /// Constructs a new `SliceArray`. pub fn try_new(child: ArrayRef, range: Range) -> VortexResult { - Array::try_from_data(SliceData::try_new(child, range)?) + let len = range.len(); + let dtype = child.dtype().clone(); + let data = SliceData::try_new(child, range)?; + Array::try_from_parts(ArrayNew::new(Slice, dtype, len, data)) } /// Constructs a new `SliceArray`. pub fn new(child: ArrayRef, range: Range) -> Self { - Array::try_from_data(SliceData::new(child, range)) + let len = range.len(); + let dtype = child.dtype().clone(); + let data = SliceData::new(child, range); + Array::try_from_parts(ArrayNew::new(Slice, dtype, len, data)) .vortex_expect("SliceData is always valid") } } diff --git a/vortex-array/src/arrays/slice/vtable.rs b/vortex-array/src/arrays/slice/vtable.rs index c5fa9241001..b8c59c68da4 100644 --- a/vortex-array/src/arrays/slice/vtable.rs +++ b/vortex-array/src/arrays/slice/vtable.rs @@ -51,29 +51,13 @@ impl Slice { impl VTable for Slice { type ArrayData = SliceData; - type Metadata = SliceMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &SliceData) -> &Self { - &Slice - } fn id(&self) -> ArrayId { Slice::ID } - fn len(array: &SliceData) -> usize { - array.range.len() - } - - fn dtype(array: &SliceData) -> &DType { - array.child().dtype() - } - - fn stats(array: &SliceData) -> &ArrayStats { - &array.stats - } - fn array_hash(array: &SliceData, state: &mut H, precision: Precision) { array.child().array_hash(state, precision); array.range.start.hash(state); @@ -104,35 +88,21 @@ impl VTable for Slice { SLOT_NAMES[idx].to_string() } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(SliceMetadata(array.range.clone())) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { // TODO(joe): make this configurable vortex_bail!("Slice array is not serializable") } fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - vortex_bail!("Slice array is not serializable") - } - - fn build( + &self, dtype: &DType, - len: usize, - metadata: &SliceMetadata, + len: usize, _metadata: &[u8], + _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { - assert_eq!(len, metadata.0.len()); - let child = children.get(0, dtype, metadata.0.end)?; - SliceData::try_new(child, metadata.0.clone()) + vortex_bail!("Slice array is not serializable") } fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { diff --git a/vortex-array/src/arrays/struct_/array.rs b/vortex-array/src/arrays/struct_/array.rs index f8ce8a54bae..da08ba9f78a 100644 --- a/vortex-array/src/arrays/struct_/array.rs +++ b/vortex-array/src/arrays/struct_/array.rs @@ -13,6 +13,7 @@ use vortex_error::vortex_err; use crate::ArrayRef; use crate::IntoArray; use crate::array::Array; +use crate::array::ArrayNew; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Struct; @@ -529,7 +530,10 @@ impl Array { length: usize, validity: Validity, ) -> Self { - Array::try_from_data(StructData::new(names, fields, length, validity)) + let data = StructData::new(names, fields, length, validity); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Struct, dtype, len, data)) .vortex_expect("StructData is always valid") } @@ -540,7 +544,10 @@ impl Array { length: usize, validity: Validity, ) -> VortexResult { - Array::try_from_data(StructData::try_new(names, fields, length, validity)?) + let data = StructData::try_new(names, fields, length, validity)?; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Struct, dtype, len, data)) } /// Creates a new `StructArray` without validation. @@ -554,7 +561,10 @@ impl Array { length: usize, validity: Validity, ) -> Self { - Array::try_from_data(unsafe { StructData::new_unchecked(fields, dtype, length, validity) }) + let data = unsafe { StructData::new_unchecked(fields, dtype, length, validity) }; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Struct, dtype, len, data)) .vortex_expect("StructData is always valid") } @@ -565,14 +575,18 @@ impl Array { length: usize, validity: Validity, ) -> VortexResult { - Array::try_from_data(StructData::try_new_with_dtype( - fields, dtype, length, validity, - )?) + let data = StructData::try_new_with_dtype(fields, dtype, length, validity)?; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Struct, dtype, len, data)) } /// Construct a `StructArray` from named fields. pub fn from_fields>(items: &[(N, ArrayRef)]) -> VortexResult { - Array::try_from_data(StructData::from_fields(items)?) + let data = StructData::from_fields(items)?; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Struct, dtype, len, data)) } /// Decompose this struct array into its constituent parts. @@ -589,19 +603,28 @@ impl Array { iter: T, validity: Validity, ) -> VortexResult { - Array::try_from_data(StructData::try_from_iter_with_validity(iter, validity)?) + let data = StructData::try_from_iter_with_validity(iter, validity)?; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Struct, dtype, len, data)) } /// Create a `StructArray` from an iterator of (name, array) pairs. pub fn try_from_iter, A: IntoArray, T: IntoIterator>( iter: T, ) -> VortexResult { - Array::try_from_data(StructData::try_from_iter(iter)?) + let data = StructData::try_from_iter(iter)?; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Struct, dtype, len, data)) } /// Create a fieldless `StructArray` with the given length. pub fn new_fieldless_with_len(len: usize) -> Self { - Array::try_from_data(StructData::new_fieldless_with_len(len)) + let data = StructData::new_fieldless_with_len(len); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Struct, dtype, len, data)) .vortex_expect("StructData is always valid") } } diff --git a/vortex-array/src/arrays/struct_/vtable/mod.rs b/vortex-array/src/arrays/struct_/vtable/mod.rs index 819db769eff..ead30676889 100644 --- a/vortex-array/src/arrays/struct_/vtable/mod.rs +++ b/vortex-array/src/arrays/struct_/vtable/mod.rs @@ -40,29 +40,13 @@ vtable!(Struct, Struct, StructData); impl VTable for Struct { type ArrayData = StructData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Struct - } fn id(&self) -> ArrayId { Self::ID } - fn len(array: &StructData) -> usize { - array.len - } - - fn dtype(array: &StructData) -> &DType { - &array.dtype - } - - fn stats(array: &StructData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &StructData, state: &mut H, precision: Precision) { for field in array.iter_unmasked_fields() { field.array_hash(state, precision); @@ -91,31 +75,20 @@ impl VTable for Struct { vortex_panic!("StructArray buffer_name index {idx} out of bounds") } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) - } - - fn build( + &self, dtype: &DType, - len: usize, - _metadata: &Self::Metadata, + len: usize, metadata: &[u8], + _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + ::deserialize(metadata)?; let DType::Struct(struct_dtype, nullability) = dtype else { vortex_bail!("Expected struct dtype, found {:?}", dtype) }; diff --git a/vortex-array/src/arrays/varbin/array.rs b/vortex-array/src/arrays/varbin/array.rs index c50c06285ef..1acb47fbce8 100644 --- a/vortex-array/src/arrays/varbin/array.rs +++ b/vortex-array/src/arrays/varbin/array.rs @@ -12,6 +12,7 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::ToCanonical; use crate::array::Array; +use crate::array::ArrayNew; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::VarBin; @@ -382,7 +383,10 @@ impl VarBinData { /// Forwarding constructors for `VarBinArray` (= `Array`). impl Array { pub fn from_vec>(vec: Vec, dtype: DType) -> Self { - Array::try_from_data(VarBinData::from_vec(vec, dtype)) + let data = VarBinData::from_vec(vec, dtype); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") } @@ -394,7 +398,10 @@ impl Array { iter: I, dtype: DType, ) -> Self { - Array::try_from_data(VarBinData::from_iter(iter, dtype)) + let data = VarBinData::from_iter(iter, dtype); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") } @@ -402,7 +409,10 @@ impl Array { iter: I, dtype: DType, ) -> Self { - Array::try_from_data(VarBinData::from_iter_nonnull(iter, dtype)) + let data = VarBinData::from_iter_nonnull(iter, dtype); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") } @@ -472,7 +482,10 @@ impl VarBinData { impl Array { /// Creates a new `VarBinArray`. pub fn new(offsets: ArrayRef, bytes: ByteBuffer, dtype: DType, validity: Validity) -> Self { - Array::try_from_data(VarBinData::new(offsets, bytes, dtype, validity)) + let data = VarBinData::new(offsets, bytes, dtype, validity); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") } @@ -487,7 +500,10 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Array::try_from_data(unsafe { VarBinData::new_unchecked(offsets, bytes, dtype, validity) }) + let data = unsafe { VarBinData::new_unchecked(offsets, bytes, dtype, validity) }; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") } @@ -502,9 +518,12 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Array::try_from_data(unsafe { + let data = unsafe { VarBinData::new_unchecked_from_handle(offsets, bytes, dtype, validity) - }) + }; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") } @@ -515,7 +534,10 @@ impl Array { dtype: DType, validity: Validity, ) -> VortexResult { - Array::try_from_data(VarBinData::try_new(offsets, bytes, dtype, validity)?) + let data = VarBinData::try_new(offsets, bytes, dtype, validity)?; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) } } @@ -595,76 +617,120 @@ impl<'a> FromIterator> for VarBinData { impl From> for Array { fn from(value: Vec<&[u8]>) -> Self { - Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") + let data = VarBinData::from(value); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + .vortex_expect("VarBinData is always valid") } } impl From>> for Array { fn from(value: Vec>) -> Self { - Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") + let data = VarBinData::from(value); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + .vortex_expect("VarBinData is always valid") } } impl From> for Array { fn from(value: Vec) -> Self { - Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") + let data = VarBinData::from(value); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + .vortex_expect("VarBinData is always valid") } } impl From> for Array { fn from(value: Vec<&str>) -> Self { - Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") + let data = VarBinData::from(value); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + .vortex_expect("VarBinData is always valid") } } impl From>> for Array { fn from(value: Vec>) -> Self { - Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") + let data = VarBinData::from(value); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + .vortex_expect("VarBinData is always valid") } } impl From>>> for Array { fn from(value: Vec>>) -> Self { - Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") + let data = VarBinData::from(value); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + .vortex_expect("VarBinData is always valid") } } impl From>> for Array { fn from(value: Vec>) -> Self { - Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") + let data = VarBinData::from(value); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + .vortex_expect("VarBinData is always valid") } } impl From>> for Array { fn from(value: Vec>) -> Self { - Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") + let data = VarBinData::from(value); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + .vortex_expect("VarBinData is always valid") } } impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Array::try_from_data(>::from_iter(iter)) + let data = >::from_iter(iter); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) .vortex_expect(" is always valid") } } impl FromIterator>> for Array { fn from_iter>>>(iter: T) -> Self { - Array::try_from_data(>::from_iter(iter)) + let data = >::from_iter(iter); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) .vortex_expect(" is always valid") } } impl FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Array::try_from_data(>::from_iter(iter)) + let data = >::from_iter(iter); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) .vortex_expect(" is always valid") } } impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Array::try_from_data(>::from_iter(iter)) + let data = >::from_iter(iter); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) .vortex_expect(" is always valid") } } diff --git a/vortex-array/src/arrays/varbin/builder.rs b/vortex-array/src/arrays/varbin/builder.rs index 78675e337d7..e65bd6dc37d 100644 --- a/vortex-array/src/arrays/varbin/builder.rs +++ b/vortex-array/src/arrays/varbin/builder.rs @@ -105,7 +105,9 @@ impl VarBinBuilder { offsets.statistics().compute_is_sorted().unwrap_or(false), "VarBinBuilder offsets must be sorted" ); - Primitive::stats(&offsets).set(Stat::IsSorted, Precision::Exact(true.into())); + offsets + .statistics() + .set(Stat::IsSorted, Precision::Exact(true.into())); // SAFETY: The builder maintains all invariants: // - Offsets are monotonically increasing starting from 0 (guaranteed by builder logic). diff --git a/vortex-array/src/arrays/varbin/vtable/mod.rs b/vortex-array/src/arrays/varbin/vtable/mod.rs index 8b8664ce506..bdcbeb5f200 100644 --- a/vortex-array/src/arrays/varbin/vtable/mod.rs +++ b/vortex-array/src/arrays/varbin/vtable/mod.rs @@ -54,29 +54,13 @@ pub struct VarBinMetadata { impl VTable for VarBin { type ArrayData = VarBinData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &VarBinData) -> &Self { - &VarBin - } fn id(&self) -> ArrayId { Self::ID } - fn len(array: &VarBinData) -> usize { - array.offsets().len().saturating_sub(1) - } - - fn dtype(array: &VarBinData) -> &DType { - &array.dtype - } - - fn stats(array: &VarBinData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &VarBinData, state: &mut H, precision: Precision) { array.bytes().array_hash(state, precision); array.offsets().array_hash(state, precision); @@ -107,36 +91,24 @@ impl VTable for VarBin { } } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(VarBinMetadata { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some(ProstMetadata(VarBinMetadata { offsets_ptype: PType::try_from(array.offsets().dtype()) .vortex_expect("Must be a valid PType") as i32, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + }) + .serialize())) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata(ProstMetadata::::deserialize( - bytes, - )?)) - } - - fn build( + &self, dtype: &DType, - len: usize, - metadata: &Self::Metadata, + len: usize, metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = ProstMetadata::::deserialize(metadata)?; let validity = if children.len() == 1 { Validity::from(dtype.nullability()) } else if children.len() == 2 { diff --git a/vortex-array/src/arrays/varbinview/array.rs b/vortex-array/src/arrays/varbinview/array.rs index d7228c65ed0..ca9f2d990f5 100644 --- a/vortex-array/src/arrays/varbinview/array.rs +++ b/vortex-array/src/arrays/varbinview/array.rs @@ -17,6 +17,7 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::array::Array; +use crate::array::ArrayNew; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::VarBinView; @@ -561,31 +562,46 @@ impl Array { iter: I, dtype: DType, ) -> Self { - Array::try_from_data(VarBinViewData::from_iter(iter, dtype)) + let data = VarBinViewData::from_iter(iter, dtype); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") } pub fn from_iter_str, I: IntoIterator>(iter: I) -> Self { - Array::try_from_data(VarBinViewData::from_iter_str(iter)) + let data = VarBinViewData::from_iter_str(iter); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") } pub fn from_iter_nullable_str, I: IntoIterator>>( iter: I, ) -> Self { - Array::try_from_data(VarBinViewData::from_iter_nullable_str(iter)) + let data = VarBinViewData::from_iter_nullable_str(iter); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") } pub fn from_iter_bin, I: IntoIterator>(iter: I) -> Self { - Array::try_from_data(VarBinViewData::from_iter_bin(iter)) + let data = VarBinViewData::from_iter_bin(iter); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") } pub fn from_iter_nullable_bin, I: IntoIterator>>( iter: I, ) -> Self { - Array::try_from_data(VarBinViewData::from_iter_nullable_bin(iter)) + let data = VarBinViewData::from_iter_nullable_bin(iter); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") } @@ -596,7 +612,10 @@ impl Array { dtype: DType, validity: Validity, ) -> VortexResult { - Array::try_from_data(VarBinViewData::try_new(views, buffers, dtype, validity)?) + let data = VarBinViewData::try_new(views, buffers, dtype, validity)?; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) } /// Creates a new `VarBinViewArray` without validation. @@ -610,9 +629,10 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Array::try_from_data(unsafe { - VarBinViewData::new_unchecked(views, buffers, dtype, validity) - }) + let data = unsafe { VarBinViewData::new_unchecked(views, buffers, dtype, validity) }; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") } @@ -623,7 +643,10 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Array::try_from_data(VarBinViewData::new_handle(views, buffers, dtype, validity)) + let data = VarBinViewData::new_handle(views, buffers, dtype, validity); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") } @@ -638,9 +661,12 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Array::try_from_data(unsafe { + let data = unsafe { VarBinViewData::new_handle_unchecked(views, buffers, dtype, validity) - }) + }; + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") } } @@ -673,28 +699,40 @@ impl<'a> FromIterator> for VarBinViewData { impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Array::try_from_data(>::from_iter(iter)) + let data = >::from_iter(iter); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) .vortex_expect(" is always valid") } } impl FromIterator>> for Array { fn from_iter>>>(iter: T) -> Self { - Array::try_from_data(>::from_iter(iter)) + let data = >::from_iter(iter); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) .vortex_expect(" is always valid") } } impl FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Array::try_from_data(>::from_iter(iter)) + let data = >::from_iter(iter); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) .vortex_expect(" is always valid") } } impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Array::try_from_data(>::from_iter(iter)) + let data = >::from_iter(iter); + let dtype = data.dtype().clone(); + let len = data.len(); + Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) .vortex_expect(" is always valid") } } diff --git a/vortex-array/src/arrays/varbinview/vtable/mod.rs b/vortex-array/src/arrays/varbinview/vtable/mod.rs index 2efb586e619..78ea7d3d092 100644 --- a/vortex-array/src/arrays/varbinview/vtable/mod.rs +++ b/vortex-array/src/arrays/varbinview/vtable/mod.rs @@ -50,29 +50,13 @@ impl VarBinView { impl VTable for VarBinView { type ArrayData = VarBinViewData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &VarBinViewData) -> &Self { - &VarBinView - } fn id(&self) -> ArrayId { Self::ID } - fn len(array: &VarBinViewData) -> usize { - array.views_handle().len() / size_of::() - } - - fn dtype(array: &VarBinViewData) -> &DType { - &array.dtype - } - - fn stats(array: &VarBinViewData) -> &ArrayStats { - &array.stats_set - } - fn array_hash( array: &VarBinViewData, state: &mut H, @@ -122,31 +106,20 @@ impl VTable for VarBinView { } } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) - } - - fn build( + &self, dtype: &DType, - len: usize, - _metadata: &Self::Metadata, + len: usize, metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + ::deserialize(metadata)?; let Some((views_handle, data_handles)) = buffers.split_last() else { vortex_bail!("Expected at least 1 buffer, got 0"); }; diff --git a/vortex-array/src/arrays/variant/mod.rs b/vortex-array/src/arrays/variant/mod.rs index 29fdb5edcd2..96e1015d61e 100644 --- a/vortex-array/src/arrays/variant/mod.rs +++ b/vortex-array/src/arrays/variant/mod.rs @@ -9,6 +9,7 @@ pub use self::vtable::Variant; pub use self::vtable::VariantArray; use crate::ArrayRef; use crate::array::Array; +use crate::array::ArrayNew; use crate::dtype::DType; use crate::stats::ArrayStats; @@ -65,6 +66,10 @@ impl VariantData { impl Array { /// Creates a new `VariantArray`. pub fn new(child: ArrayRef) -> Self { - Array::try_from_data(VariantData::new(child)).vortex_expect("VariantData is always valid") + let dtype = child.dtype().clone(); + let len = child.len(); + let stats = child.statistics().to_array_stats(); + Array::try_from_parts(ArrayNew::new(Variant, dtype, len, VariantData::new(child)).with_stats(stats)) + .vortex_expect("VariantData is always valid") } } diff --git a/vortex-array/src/arrays/variant/vtable/mod.rs b/vortex-array/src/arrays/variant/vtable/mod.rs index 31f52911e6d..bab4fded841 100644 --- a/vortex-array/src/arrays/variant/vtable/mod.rs +++ b/vortex-array/src/arrays/variant/vtable/mod.rs @@ -42,32 +42,14 @@ impl Variant { impl VTable for Variant { type ArrayData = VariantData; - type Metadata = EmptyMetadata; - type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Variant - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &Self::ArrayData) -> usize { - array.child().len() - } - - fn dtype(array: &Self::ArrayData) -> &DType { - array.child().dtype() - } - - fn stats(array: &Self::ArrayData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &VariantData, state: &mut H, precision: Precision) { array.child().array_hash(state, precision); } @@ -88,31 +70,20 @@ impl VTable for Variant { None } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &vortex_session::VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) - } - - fn build( + &self, dtype: &DType, - len: usize, - _metadata: &Self::Metadata, + len: usize, metadata: &[u8], + _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &vortex_session::VortexSession, ) -> VortexResult { + ::deserialize(metadata)?; vortex_ensure!(matches!(dtype, DType::Variant(_)), "Expected Variant DType"); vortex_ensure!( children.len() == 1, diff --git a/vortex-array/src/display/extractors/metadata.rs b/vortex-array/src/display/extractors/metadata.rs index a355bafe63c..20af3ead0a8 100644 --- a/vortex-array/src/display/extractors/metadata.rs +++ b/vortex-array/src/display/extractors/metadata.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::fmt; +use std::fmt::Debug; use crate::ArrayRef; use crate::display::extractor::IndentedFormatter; @@ -20,7 +21,11 @@ impl TreeExtractor for MetadataExtractor { ) -> fmt::Result { let (indent, f) = f.parts(); write!(f, "{indent}metadata: ")?; - array.metadata_fmt(f)?; + match array.metadata() { + Ok(Some(metadata)) => Debug::fmt(&metadata, f)?, + Ok(None) => write!(f, "")?, + Err(err) => write!(f, "")?, + } writeln!(f) } } diff --git a/vortex-array/src/scalar_fn/fns/select.rs b/vortex-array/src/scalar_fn/fns/select.rs index 7b323c4311c..1415aba5073 100644 --- a/vortex-array/src/scalar_fn/fns/select.rs +++ b/vortex-array/src/scalar_fn/fns/select.rs @@ -18,6 +18,9 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; +use crate::array::Array; +use crate::array::ArrayNew; +use crate::arrays::Struct; use crate::arrays::StructArray; use crate::dtype::DType; use crate::dtype::FieldName; @@ -160,7 +163,11 @@ impl ScalarFnVTable for Select { } }?; - result.into_array().execute(ctx) + let dtype = result.dtype().clone(); + let len = result.len(); + Array::try_from_parts(ArrayNew::new(Struct, dtype, len, result))? + .into_array() + .execute(ctx) } fn simplify( From c683cc50ee2648b601d5c6004276ec72154c6beb Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Thu, 2 Apr 2026 17:28:48 -0400 Subject: [PATCH 02/15] Follow up Signed-off-by: Nicholas Gates --- .../src/bitpacking/array/bitpack_compress.rs | 67 ++--- .../bitpacking/array/bitpack_decompress.rs | 32 ++- .../fastlanes/src/bitpacking/array/mod.rs | 75 +++-- .../src/bitpacking/array/unpack_iter.rs | 4 +- .../fastlanes/src/bitpacking/compute/cast.rs | 7 +- .../src/bitpacking/compute/filter.rs | 27 +- .../src/bitpacking/compute/is_constant.rs | 6 +- .../fastlanes/src/bitpacking/compute/slice.rs | 19 +- .../fastlanes/src/bitpacking/compute/take.rs | 26 +- .../fastlanes/src/bitpacking/vtable/mod.rs | 136 +++++---- .../src/bitpacking/vtable/operations.rs | 4 +- .../src/bitpacking/vtable/validity.rs | 2 +- .../src/delta/array/delta_compress.rs | 20 +- encodings/fastlanes/src/delta/array/mod.rs | 77 +++--- encodings/fastlanes/src/delta/compute/cast.rs | 17 +- encodings/fastlanes/src/delta/vtable/mod.rs | 77 +++--- .../fastlanes/src/delta/vtable/operations.rs | 9 +- encodings/fastlanes/src/delta/vtable/slice.rs | 13 +- .../fastlanes/src/for/array/for_compress.rs | 30 +- .../fastlanes/src/for/array/for_decompress.rs | 4 +- encodings/fastlanes/src/for/array/mod.rs | 54 ++-- encodings/fastlanes/src/for/compute/cast.rs | 10 +- .../fastlanes/src/for/compute/compare.rs | 53 ++-- .../fastlanes/src/for/compute/is_sorted.rs | 10 +- encodings/fastlanes/src/for/compute/mod.rs | 25 +- encodings/fastlanes/src/for/vtable/mod.rs | 61 ++-- encodings/fastlanes/src/for/vtable/rules.rs | 13 +- encodings/fastlanes/src/for/vtable/slice.rs | 13 +- encodings/fastlanes/src/rle/array/mod.rs | 85 +++--- .../fastlanes/src/rle/array/rle_compress.rs | 11 +- encodings/fastlanes/src/rle/compute/cast.rs | 13 +- encodings/fastlanes/src/rle/kernel.rs | 13 +- encodings/fastlanes/src/rle/vtable/mod.rs | 79 +++--- .../fastlanes/src/rle/vtable/validity.rs | 24 +- encodings/fsst/src/array.rs | 196 ++++++------- encodings/fsst/src/canonical.rs | 6 +- encodings/fsst/src/compress.rs | 9 +- encodings/fsst/src/compute/cast.rs | 3 +- encodings/fsst/src/compute/compare.rs | 8 +- encodings/fsst/src/compute/filter.rs | 3 +- encodings/fsst/src/compute/mod.rs | 3 +- encodings/fsst/src/slice.rs | 3 +- encodings/parquet-variant/src/array.rs | 104 +++++-- encodings/parquet-variant/src/kernel.rs | 7 +- encodings/parquet-variant/src/lib.rs | 1 - encodings/parquet-variant/src/operations.rs | 2 +- encodings/parquet-variant/src/vtable.rs | 110 +++----- encodings/zstd/src/array.rs | 261 ++++++++++-------- encodings/zstd/src/compute/cast.rs | 32 ++- encodings/zstd/src/slice.rs | 5 +- encodings/zstd/src/test.rs | 8 +- encodings/zstd/src/zstd_buffers.rs | 222 +++++++-------- vortex-btrblocks/src/schemes/string.rs | 2 +- vortex-cuda/src/hybrid_dispatch/mod.rs | 3 +- .../src/kernel/encodings/zstd_buffers.rs | 9 +- 55 files changed, 1041 insertions(+), 1072 deletions(-) diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index 43ae4e410c5..6c87ded63ac 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -23,8 +23,8 @@ use vortex_error::vortex_bail; use vortex_mask::AllOr; use vortex_mask::Mask; +use crate::BitPacked; use crate::BitPackedArray; -use crate::BitPackedData; use crate::bitpack_decompress; pub fn bitpack_to_best_bit_width(array: &PrimitiveArray) -> VortexResult { @@ -71,27 +71,16 @@ pub fn bitpack_encode( .transpose()? .flatten(); - // SAFETY: all components validated above - let bitpacked = unsafe { - BitPackedData::new_unchecked( - BufferHandle::new_host(packed), - array.dtype().clone(), - array.validity(), - patches, - bit_width, - array.len(), - 0, - ) - }; - let bitpacked = - BitPackedArray::try_from_data(bitpacked).vortex_expect("BitPackedData is always valid"); - { - let bp_ref = bitpacked.clone().into_array(); - bitpacked - .stats_set - .to_ref(&bp_ref) - .inherit_from(array.statistics()); - } + let bitpacked = BitPacked::try_new( + BufferHandle::new_host(packed), + array.ptype(), + array.validity(), + patches, + bit_width, + array.len(), + 0, + )?; + bitpacked.statistics().inherit_from(array.statistics()); Ok(bitpacked) } @@ -110,28 +99,18 @@ pub unsafe fn bitpack_encode_unchecked( // SAFETY: non-negativity of input checked by caller. let packed = unsafe { bitpack_unchecked(&array, bit_width) }; - // SAFETY: checked by bitpack_unchecked - let data = unsafe { - BitPackedData::new_unchecked( - BufferHandle::new_host(packed), - array.dtype().clone(), - array.validity(), - None, - bit_width, - array.len(), - 0, - ) - }; - let bitpacked = - BitPackedArray::try_from_data(data).vortex_expect("BitPackedData is always valid"); - { - let bp_ref = bitpacked.clone().into_array(); - let arr_ref = array.into_array(); - bitpacked - .stats_set - .to_ref(&bp_ref) - .inherit_from(arr_ref.statistics()); - } + let arr_ref = array.clone().into_array(); + let bitpacked = BitPacked::try_new( + BufferHandle::new_host(packed), + array.ptype(), + array.validity(), + None, + bit_width, + array.len(), + 0, + ) + .vortex_expect("bitpacked array construction should succeed"); + bitpacked.statistics().inherit_from(arr_ref.statistics()); Ok(bitpacked) } diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs index 4f31154d598..989fe10b619 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs @@ -4,6 +4,7 @@ use fastlanes::BitPacking; use itertools::Itertools; use num_traits::AsPrimitive; +use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::arrays::PrimitiveArray; use vortex_array::builders::ArrayBuilder; @@ -17,28 +18,31 @@ use vortex_array::scalar::Scalar; use vortex_error::VortexExpect; use vortex_error::VortexResult; -use crate::BitPackedData; -use crate::unpack_iter::BitPacked; +use crate::BitPacked; +use crate::unpack_iter::BitPacked as BitPackedUnpack; /// Unpacks a bit-packed array into a primitive array. -pub fn unpack_array(array: &BitPackedData, ctx: &mut ExecutionCtx) -> VortexResult { - match_each_integer_ptype!(array.ptype(), |P| { +pub fn unpack_array( + array: ArrayView<'_, BitPacked>, + ctx: &mut ExecutionCtx, +) -> VortexResult { + match_each_integer_ptype!(array.dtype().as_ptype(), |P| { unpack_primitive_array::

(array, ctx) }) } -pub fn unpack_primitive_array( - array: &BitPackedData, +pub fn unpack_primitive_array( + array: ArrayView<'_, BitPacked>, ctx: &mut ExecutionCtx, ) -> VortexResult { let mut builder = PrimitiveBuilder::with_capacity(array.dtype().nullability(), array.len()); - unpack_into_primitive_builder::(array, &mut builder, ctx)?; + unpack_into_primitive_builder::(&array, &mut builder, ctx)?; assert_eq!(builder.len(), array.len()); Ok(builder.finish_into_primitive()) } -pub(crate) fn unpack_into_primitive_builder( - array: &BitPackedData, +pub(crate) fn unpack_into_primitive_builder( + array: &ArrayView<'_, BitPacked>, // TODO(ngates): do we want to use fastlanes alignment for this buffer? builder: &mut PrimitiveBuilder, ctx: &mut ExecutionCtx, @@ -53,16 +57,16 @@ pub(crate) fn unpack_into_primitive_builder( // SAFETY: We later initialize the the uninitialized range of values with `copy_from_slice`. unsafe { // Append a dense null Mask. - uninit_range.append_mask(array.validity_mask()); + uninit_range.append_mask(array.validity_mask(array.len(), array.dtype().nullability())); } // SAFETY: `decode_into` will initialize all values in this range. let uninit_slice = unsafe { uninit_range.slice_uninit_mut(0, array.len()) }; - let mut bit_packed_iter = array.unpacked_chunks(); + let mut bit_packed_iter = array.unpacked_chunks(array.dtype(), array.len()); bit_packed_iter.decode_into(uninit_slice); - if let Some(ref patches) = array.patches() { + if let Some(ref patches) = array.patches(array.len()) { apply_patches_to_uninit_range(&mut uninit_range, patches, ctx)?; }; @@ -106,9 +110,9 @@ pub fn apply_patches_to_uninit_range_fn T>( Ok(()) } -pub fn unpack_single(array: &BitPackedData, index: usize) -> Scalar { +pub fn unpack_single(array: ArrayView<'_, BitPacked>, index: usize) -> Scalar { let bit_width = array.bit_width() as usize; - let ptype = array.ptype(); + let ptype = array.dtype().as_ptype(); // let packed = array.packed().into_primitive()?; let index_in_encoded = index + array.offset() as usize; let scalar: Scalar = match_each_unsigned_integer_ptype!(ptype.to_unsigned(), |P| { diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index f54e788650b..00b488a96ca 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -8,9 +8,9 @@ use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; +use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::patches::Patches; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable::child_to_validity; use vortex_array::vtable::validity_to_child; @@ -59,15 +59,12 @@ pub struct BitPackedData { /// The offset within the first block (created with a slice). /// 0 <= offset < 1024 pub(super) offset: u16, - pub(super) len: usize, - pub(super) dtype: DType, pub(super) bit_width: u8, pub(super) packed: BufferHandle, /// The offset metadata from patches, needed to reconstruct Patches from slots. pub(super) patch_offset: Option, /// The offset_within_chunk metadata from patches. pub(super) patch_offset_within_chunk: Option, - pub(super) stats_set: ArrayStats, } impl BitPackedData { @@ -93,7 +90,6 @@ impl BitPackedData { /// bit-packed array. pub(crate) unsafe fn new_unchecked( packed: BufferHandle, - dtype: DType, validity: Validity, patches: Option, bit_width: u8, @@ -109,13 +105,10 @@ impl BitPackedData { Self { slots, offset, - len, - dtype, bit_width, packed, patch_offset, patch_offset_within_chunk, - stats_set: Default::default(), } } @@ -176,16 +169,28 @@ impl BitPackedData { offset, )?; - let dtype = DType::Primitive(ptype, validity.nullability()); - // SAFETY: all components validated above unsafe { Ok(Self::new_unchecked( - packed, dtype, validity, patches, bit_width, length, offset, + packed, validity, patches, bit_width, length, offset, )) } } + pub(crate) fn validate_against_outer(&self, dtype: &DType, len: usize) -> VortexResult<()> { + let validity = self.validity(dtype.nullability()); + let patches = self.patches(len); + Self::validate( + &self.packed, + dtype.as_ptype(), + &validity, + patches.as_ref(), + self.bit_width, + len, + self.offset, + ) + } + fn validate( packed: &BufferHandle, ptype: PType, @@ -247,31 +252,13 @@ impl BitPackedData { Ok(()) } - /// Returns the length of the array. - #[inline] - pub fn len(&self) -> usize { - self.len - } - - /// Returns `true` if the array is empty. - #[inline] - pub fn is_empty(&self) -> bool { - self.len == 0 - } - - /// Returns the dtype of the array. - #[inline] - pub fn dtype(&self) -> &DType { - &self.dtype - } - /// Returns the validity as a [`Mask`](vortex_mask::Mask). - pub fn validity_mask(&self) -> vortex_mask::Mask { - self.validity().to_mask(self.len()) + pub fn validity_mask(&self, len: usize, nullability: Nullability) -> vortex_mask::Mask { + self.validity(nullability).to_mask(len) } - pub fn ptype(&self) -> PType { - self.dtype.as_ptype() + pub fn ptype(&self, dtype: &DType) -> PType { + dtype.as_ptype() } /// Underlying bit packed values as byte array @@ -295,13 +282,13 @@ impl BitPackedData { } /// Accessor for bit unpacked chunks - pub fn unpacked_chunks(&self) -> BitUnpackedChunks { + pub fn unpacked_chunks(&self, dtype: &DType, len: usize) -> BitUnpackedChunks { assert_eq!( T::PTYPE, - self.ptype(), + self.ptype(dtype), "Requested type doesn't match the array ptype" ); - BitUnpackedChunks::new(self) + BitUnpackedChunks::new(self, len) } /// Bit-width of the packed values @@ -315,7 +302,7 @@ impl BitPackedData { /// Reconstructs a `Patches` from the stored slots and patch metadata. /// If present, patches MUST be a `SparseArray` with equal-length to this array, and whose /// indices indicate the locations of patches. The indices must have non-zero length. - pub fn patches(&self) -> Option { + pub fn patches(&self, len: usize) -> Option { match ( &self.slots[PATCH_INDICES_SLOT], &self.slots[PATCH_VALUES_SLOT], @@ -326,7 +313,7 @@ impl BitPackedData { .vortex_expect("has patch slots but no patch_offset"); Some(unsafe { Patches::new_unchecked( - self.len, + len, patch_offset, indices.clone(), values.clone(), @@ -340,8 +327,8 @@ impl BitPackedData { } /// Returns the validity, reconstructed from the stored slot. - pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + pub fn validity(&self, nullability: Nullability) -> Validity { + child_to_validity(&self.slots[VALIDITY_SLOT], nullability) } pub fn replace_patches(&mut self, patches: Option) { @@ -392,13 +379,13 @@ impl BitPackedData { (1 << self.bit_width()) - 1 } - pub fn into_parts(self) -> BitPackedArrayParts { - let patches = self.patches(); - let validity = self.validity(); + pub fn into_parts(self, len: usize, nullability: Nullability) -> BitPackedArrayParts { + let patches = self.patches(len); + let validity = self.validity(nullability); BitPackedArrayParts { offset: self.offset, bit_width: self.bit_width, - len: self.len, + len, packed: self.packed, patches, validity, diff --git a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs index d14d218401c..e8d8ac059bd 100644 --- a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs +++ b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs @@ -89,13 +89,13 @@ pub struct UnpackedChunks> { pub type BitUnpackedChunks = UnpackedChunks; impl BitUnpackedChunks { - pub fn new(array: &BitPackedData) -> Self { + pub fn new(array: &BitPackedData, len: usize) -> Self { Self::new_with_strategy( BitPackingStrategy, array.packed().clone().unwrap_host(), array.bit_width() as usize, array.offset() as usize, - array.len(), + len, ) } diff --git a/encodings/fastlanes/src/bitpacking/compute/cast.rs b/encodings/fastlanes/src/bitpacking/compute/cast.rs index 43cbc38d455..b7e6a7b63da 100644 --- a/encodings/fastlanes/src/bitpacking/compute/cast.rs +++ b/encodings/fastlanes/src/bitpacking/compute/cast.rs @@ -10,21 +10,20 @@ use vortex_array::patches::Patches; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; -use crate::BitPackedData; use crate::bitpacking::BitPacked; impl CastReduce for BitPacked { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { if array.dtype().eq_ignore_nullability(dtype) { let new_validity = array - .validity() + .validity(array.dtype().nullability()) .cast_nullability(dtype.nullability(), array.len())?; return Ok(Some( - BitPackedData::try_new( + BitPacked::try_new( array.packed().clone(), dtype.as_ptype(), new_validity, array - .patches() + .patches(array.len()) .map(|patches| { let new_values = patches.values().cast(dtype.clone())?; Patches::new( diff --git a/encodings/fastlanes/src/bitpacking/compute/filter.rs b/encodings/fastlanes/src/bitpacking/compute/filter.rs index a2887615698..b8a7f398518 100644 --- a/encodings/fastlanes/src/bitpacking/compute/filter.rs +++ b/encodings/fastlanes/src/bitpacking/compute/filter.rs @@ -58,30 +58,39 @@ impl FilterKernel for BitPacked { // If the density is high enough, then we would rather decompress the whole array and then apply // a filter over decompressing values one by one. - if values.density() > unpack_then_filter_threshold(array.ptype()) { + if values.density() > unpack_then_filter_threshold(array.dtype().as_ptype()) { return Ok(None); } // Filter and patch using the correct unsigned type for FastLanes, then cast to signed if needed. - let primitive = match_each_unsigned_integer_ptype!(array.ptype().to_unsigned(), |U| { + let primitive = match_each_unsigned_integer_ptype!(array.dtype().as_ptype().to_unsigned(), |U| { let (buffer, validity) = filter_primitive_without_patches::(&array, values)?; // reinterpret_cast for signed types. - PrimitiveArray::new(buffer, validity).reinterpret_cast(array.ptype()) + let primitive = PrimitiveArray::new(buffer, validity); + if array.dtype().as_ptype().is_signed_int() { + PrimitiveArray::from_buffer_handle( + primitive.buffer_handle().clone(), + array.dtype().as_ptype(), + primitive.validity(), + ) + } else { + primitive + } }); let patches = array - .patches() + .patches(array.len()) .map(|patches| patches.filter(&Mask::Values(values.clone()), ctx)) .transpose()? .flatten(); if let Some(patches) = patches { - let mut prim_array = PrimitiveArray::try_from_data(primitive)?; + let mut prim_array = primitive; prim_array = prim_array.patch(&patches, ctx)?; return Ok(Some(prim_array.into_array())); } - Ok(Some(PrimitiveArray::try_from_data(primitive)?.into_array())) + Ok(Some(primitive.into_array())) } } @@ -97,11 +106,13 @@ impl FilterKernel for BitPacked { /// /// Returns a tuple of (values buffer, validity mask). fn filter_primitive_without_patches( - array: &BitPackedData, + array: &ArrayView<'_, BitPacked>, selection: &Arc, ) -> VortexResult<(Buffer, Validity)> { let values = filter_with_indices(array, selection.indices()); - let validity = array.validity().filter(&Mask::Values(selection.clone()))?; + let validity = array + .validity(array.dtype().nullability()) + .filter(&Mask::Values(selection.clone()))?; Ok((values.freeze(), validity)) } diff --git a/encodings/fastlanes/src/bitpacking/compute/is_constant.rs b/encodings/fastlanes/src/bitpacking/compute/is_constant.rs index 314e4418ea1..548f897432d 100644 --- a/encodings/fastlanes/src/bitpacking/compute/is_constant.rs +++ b/encodings/fastlanes/src/bitpacking/compute/is_constant.rs @@ -43,7 +43,7 @@ impl DynAggregateKernel for BitPackedIsConstantKernel { return Ok(None); }; - let result = match_each_integer_ptype!(array.ptype(), |P| { + let result = match_each_integer_ptype!(array.dtype().as_ptype(), |P| { bitpacked_is_constant::() }>(array)? }); @@ -54,8 +54,8 @@ impl DynAggregateKernel for BitPackedIsConstantKernel { fn bitpacked_is_constant( array: ArrayView<'_, BitPacked>, ) -> VortexResult { - let mut bit_unpack_iterator = array.unpacked_chunks::(); - let patches = array.patches().map(|p| { + let mut bit_unpack_iterator = array.unpacked_chunks::(array.dtype(), array.len()); + let patches = array.patches(array.len()).map(|p| { let values = p.values().to_primitive(); let indices = p.indices().to_primitive(); let offset = p.offset(); diff --git a/encodings/fastlanes/src/bitpacking/compute/slice.rs b/encodings/fastlanes/src/bitpacking/compute/slice.rs index fe702e524b0..e5977120a85 100644 --- a/encodings/fastlanes/src/bitpacking/compute/slice.rs +++ b/encodings/fastlanes/src/bitpacking/compute/slice.rs @@ -11,7 +11,6 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::BitPacked; -use crate::BitPackedData; impl SliceReduce for BitPacked { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { @@ -24,24 +23,22 @@ impl SliceReduce for BitPacked { let encoded_start = (block_start / 8) * array.bit_width() as usize; let encoded_stop = (block_stop / 8) * array.bit_width() as usize; - // slice the buffer using the encoded start/stop values - // SAFETY: slicing packed values without decoding preserves invariants - Ok(Some(unsafe { - BitPackedData::new_unchecked( + Ok(Some( + BitPacked::try_new( array.packed().slice(encoded_start..encoded_stop), - array.dtype().clone(), - array.validity().slice(range.clone())?, + array.dtype().as_ptype(), + array.validity(array.dtype().nullability()).slice(range.clone())?, array - .patches() + .patches(array.len()) .map(|p| p.slice(range.clone())) .transpose()? .flatten(), array.bit_width(), range.len(), offset as u16, - ) - .into_array() - })) + )? + .into_array(), + )) } } diff --git a/encodings/fastlanes/src/bitpacking/compute/take.rs b/encodings/fastlanes/src/bitpacking/compute/take.rs index bf73dd1ba05..807442b41df 100644 --- a/encodings/fastlanes/src/bitpacking/compute/take.rs +++ b/encodings/fastlanes/src/bitpacking/compute/take.rs @@ -24,7 +24,6 @@ use vortex_error::VortexResult; use super::chunked_indices; use crate::BitPacked; -use crate::BitPackedData; use crate::bitpack_decompress; // TODO(connor): This is duplicated in `encodings/fastlanes/src/bitpacking/kernels/mod.rs`. @@ -48,7 +47,7 @@ impl TakeExecute for BitPacked { // NOTE: we use the unsigned PType because all values in the BitPackedArray must // be non-negative (pre-condition of creating the BitPackedArray). let ptype: PType = PType::try_from(array.dtype())?; - let validity = array.validity(); + let validity = array.validity(array.dtype().nullability()); let taken_validity = validity.take(indices)?; let indices = indices.clone().execute::(ctx)?; @@ -57,12 +56,17 @@ impl TakeExecute for BitPacked { take_primitive::(&array, &indices, taken_validity, ctx)? }) }); - Ok(Some(taken.reinterpret_cast(ptype).into_array())) + let taken = if ptype.is_signed_int() { + PrimitiveArray::from_buffer_handle(taken.buffer_handle().clone(), ptype, taken.validity()) + } else { + taken + }; + Ok(Some(taken.into_array())) } } fn take_primitive( - array: &BitPackedData, + array: &ArrayView<'_, BitPacked>, indices: &PrimitiveArray, taken_validity: Validity, ctx: &mut ExecutionCtx, @@ -128,15 +132,17 @@ fn take_primitive( } }); - let unpatched_taken = if array.ptype().is_signed_int() { - // Flip back to signed type before patching. - PrimitiveArray::try_from_data( - PrimitiveArray::new(output, taken_validity).reinterpret_cast(array.ptype()), - )? + let unpatched_taken = if array.dtype().as_ptype().is_signed_int() { + let primitive = PrimitiveArray::new(output, taken_validity); + PrimitiveArray::from_buffer_handle( + primitive.buffer_handle().clone(), + array.dtype().as_ptype(), + primitive.validity(), + ) } else { PrimitiveArray::new(output, taken_validity) }; - if let Some(patches) = array.patches() + if let Some(patches) = array.patches(array.len()) && let Some(patches) = patches.take(&indices.clone().into_array(), ctx)? { let cast_patches = patches.cast_values(unpatched_taken.dtype())?; diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs index 1e7ccd08665..7a7865dd85a 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs @@ -3,20 +3,19 @@ use std::hash::Hash; +use prost::Message; use vortex_array::AnyCanonical; use vortex_array::Array; +use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::ArrayView; -use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; -use vortex_array::SerializeMetadata; use vortex_array::buffer::BufferHandle; use vortex_array::builders::ArrayBuilder; use vortex_array::dtype::DType; @@ -27,7 +26,6 @@ use vortex_array::patches::PatchesMetadata; use vortex_array::require_patches; use vortex_array::require_validity; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; @@ -70,29 +68,15 @@ pub struct BitPackedMetadata { impl VTable for BitPacked { type ArrayData = BitPackedData; - type Metadata = ProstMetadata; - type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &BitPackedData) -> &Self { - &BitPacked - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &BitPackedData) -> usize { - array.len - } - - fn dtype(array: &BitPackedData) -> &DType { - &array.dtype - } - - fn stats(array: &BitPackedData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate_against_outer(dtype, len) } fn array_hash( @@ -103,16 +87,36 @@ impl VTable for BitPacked { array.offset.hash(state); array.bit_width.hash(state); array.packed.array_hash(state, precision); - array.patches().array_hash(state, precision); - array.validity().array_hash(state, precision); + option_array_hash(&array.slots[PATCH_INDICES_SLOT], state, precision); + option_array_hash(&array.slots[PATCH_VALUES_SLOT], state, precision); + option_array_hash(&array.slots[PATCH_CHUNK_OFFSETS_SLOT], state, precision); + option_array_hash(&array.slots[VALIDITY_SLOT], state, precision); + array.patch_offset.hash(state); + array.patch_offset_within_chunk.hash(state); } fn array_eq(array: &BitPackedData, other: &BitPackedData, precision: Precision) -> bool { array.offset == other.offset && array.bit_width == other.bit_width && array.packed.array_eq(&other.packed, precision) - && array.patches().array_eq(&other.patches(), precision) - && array.validity().array_eq(&other.validity(), precision) + && option_array_eq( + &array.slots[PATCH_INDICES_SLOT], + &other.slots[PATCH_INDICES_SLOT], + precision, + ) + && option_array_eq( + &array.slots[PATCH_VALUES_SLOT], + &other.slots[PATCH_VALUES_SLOT], + precision, + ) + && option_array_eq( + &array.slots[PATCH_CHUNK_OFFSETS_SLOT], + &other.slots[PATCH_CHUNK_OFFSETS_SLOT], + precision, + ) + && option_array_eq(&array.slots[VALIDITY_SLOT], &other.slots[VALIDITY_SLOT], precision) + && array.patch_offset == other.patch_offset + && array.patch_offset_within_chunk == other.patch_offset_within_chunk } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -167,44 +171,30 @@ impl VTable for BitPacked { Ok(()) } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(BitPackedMetadata { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some( + BitPackedMetadata { bit_width: array.bit_width() as u32, offset: array.offset() as u32, patches: array - .patches() + .patches(array.len()) .map(|p| p.to_metadata(array.len(), array.dtype())) .transpose()?, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + } + .encode_to_vec(), + )) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - let inner = as DeserializeMetadata>::deserialize(bytes)?; - Ok(ProstMetadata(inner)) - } - - /// Deserialize a BitPackedArray from its components. - /// - /// Note that the layout depends on whether patches and chunk_offsets are present: - /// - No patches: `[validity?]` - /// - With patches: `[patch_indices, patch_values, chunk_offsets?, validity?]` - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = BitPackedMetadata::decode(metadata)?; if buffers.len() != 1 { vortex_bail!("Expected 1 buffer, got {}", buffers.len()); } @@ -274,7 +264,7 @@ impl VTable for BitPacked { builder: &mut dyn ArrayBuilder, ctx: &mut ExecutionCtx, ) -> VortexResult<()> { - match_each_integer_ptype!(array.ptype(), |T| { + match_each_integer_ptype!(array.dtype().as_ptype(), |T| { unpack_into_primitive_builder::( &array, builder @@ -289,15 +279,19 @@ impl VTable for BitPacked { fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { require_patches!( array, - array.patches(), + array.patches(array.len()), PATCH_INDICES_SLOT, PATCH_VALUES_SLOT, PATCH_CHUNK_OFFSETS_SLOT ); - require_validity!(array, &array.validity(), VALIDITY_SLOT => AnyCanonical); + require_validity!( + array, + &array.validity(array.dtype().nullability()), + VALIDITY_SLOT => AnyCanonical + ); Ok(ExecutionResult::done( - unpack_array(&array, ctx)?.into_array(), + unpack_array(array.as_view(), ctx)?.into_array(), )) } @@ -317,8 +311,44 @@ pub struct BitPacked; impl BitPacked { pub const ID: ArrayId = ArrayId::new_ref("fastlanes.bitpacked"); + pub fn try_new( + packed: BufferHandle, + ptype: PType, + validity: Validity, + patches: Option, + bit_width: u8, + len: usize, + offset: u16, + ) -> VortexResult { + let dtype = DType::Primitive(ptype, validity.nullability()); + let data = BitPackedData::try_new(packed, ptype, validity, patches, bit_width, len, offset)?; + Array::try_from_parts(ArrayNew::new(BitPacked, dtype, len, data)) + } + /// Encode an array into a bitpacked representation with the given bit width. pub fn encode(array: &ArrayRef, bit_width: u8) -> VortexResult { BitPackedData::encode(array, bit_width) } } + +fn option_array_hash( + array: &Option, + state: &mut H, + precision: Precision, +) { + match array { + Some(array) => { + true.hash(state); + array.array_hash(state, precision); + } + None => false.hash(state), + } +} + +fn option_array_eq(lhs: &Option, rhs: &Option, precision: Precision) -> bool { + match (lhs, rhs) { + (Some(lhs), Some(rhs)) => lhs.array_eq(rhs, precision), + (None, None) => true, + _ => false, + } +} diff --git a/encodings/fastlanes/src/bitpacking/vtable/operations.rs b/encodings/fastlanes/src/bitpacking/vtable/operations.rs index 8dcfa18d08d..4c9850a91e3 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/operations.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/operations.rs @@ -16,12 +16,12 @@ impl OperationsVTable for BitPacked { _ctx: &mut ExecutionCtx, ) -> VortexResult { Ok( - if let Some(patches) = array.patches() + if let Some(patches) = array.patches(array.len()) && let Some(patch) = patches.get_patched(index)? { patch } else { - bitpack_decompress::unpack_single(&array, index) + bitpack_decompress::unpack_single(array, index) }, ) } diff --git a/encodings/fastlanes/src/bitpacking/vtable/validity.rs b/encodings/fastlanes/src/bitpacking/vtable/validity.rs index c9b555eb92c..2313e0112eb 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/validity.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/validity.rs @@ -10,6 +10,6 @@ use crate::BitPacked; impl ValidityVTable for BitPacked { fn validity(array: ArrayView<'_, BitPacked>) -> VortexResult { - Ok(array.data().validity()) + Ok(array.data().validity(array.dtype().nullability())) } } diff --git a/encodings/fastlanes/src/delta/array/delta_compress.rs b/encodings/fastlanes/src/delta/array/delta_compress.rs index 3f7a75d3a82..f12bf13b674 100644 --- a/encodings/fastlanes/src/delta/array/delta_compress.rs +++ b/encodings/fastlanes/src/delta/array/delta_compress.rs @@ -104,8 +104,8 @@ mod tests { use vortex_error::VortexResult; use vortex_session::VortexSession; + use crate::Delta; use crate::DeltaArray; - use crate::DeltaData; use crate::bitpack_compress::bitpack_encode; use crate::delta::array::delta_decompress::delta_decompress; use crate::delta_compress; @@ -120,10 +120,7 @@ mod tests { (0u32..10_000).map(|i| (i % 2 == 0).then_some(i)), ))] fn test_compress(#[case] array: PrimitiveArray) -> VortexResult<()> { - let delta = DeltaArray::try_from_data(DeltaData::try_from_primitive_array( - &array, - &mut SESSION.create_execution_ctx(), - )?)?; + let delta = Delta::try_from_primitive_array(&array, &mut SESSION.create_execution_ctx())?; assert_eq!(delta.len(), array.len()); let decompressed = delta_decompress(&delta, &mut SESSION.create_execution_ctx())?; assert_arrays_eq!(decompressed, array); @@ -140,16 +137,9 @@ mod tests { ); let (bases, deltas) = delta_compress(&array, &mut SESSION.create_execution_ctx()).unwrap(); let bitpacked_deltas = bitpack_encode(&deltas, 1, None).unwrap(); - let packed_delta = DeltaArray::try_from_data( - DeltaData::try_new( - bases.into_array(), - bitpacked_deltas.into_array(), - 0, - array.len(), - ) - .unwrap(), - ) - .vortex_expect("DeltaData is always valid"); + let packed_delta = + Delta::try_new(bases.into_array(), bitpacked_deltas.into_array(), 0, array.len()) + .vortex_expect("Delta array construction should succeed"); assert_arrays_eq!(packed_delta.as_array().to_primitive(), array); } } diff --git a/encodings/fastlanes/src/delta/array/mod.rs b/encodings/fastlanes/src/delta/array/mod.rs index 048b860e000..8749f93d4c4 100644 --- a/encodings/fastlanes/src/delta/array/mod.rs +++ b/encodings/fastlanes/src/delta/array/mod.rs @@ -9,7 +9,6 @@ use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_array::match_each_unsigned_integer_ptype; -use vortex_array::stats::ArrayStats; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; @@ -37,11 +36,11 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["bases", "deltas"]; /// use vortex_array::VortexSessionExecute; /// use vortex_array::session::ArraySession; /// use vortex_session::VortexSession; -/// use vortex_fastlanes::DeltaData; +/// use vortex_fastlanes::Delta; /// /// let session = VortexSession::empty().with::(); /// let primitive = PrimitiveArray::from_iter([1_u32, 2, 3, 5, 10, 11]); -/// let array = DeltaData::try_from_primitive_array(&primitive, &mut session.create_execution_ctx()).unwrap(); +/// let array = Delta::try_from_primitive_array(&primitive, &mut session.create_execution_ctx()).unwrap(); /// ``` /// /// # Details @@ -66,14 +65,11 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["bases", "deltas"]; #[derive(Clone, Debug)] pub struct DeltaData { pub(super) offset: usize, - pub(super) len: usize, - pub(super) dtype: DType, pub(super) slots: Vec>, - pub(super) stats_set: ArrayStats, } impl DeltaData { - pub fn try_from_primitive_array( + pub(crate) fn try_from_primitive_array( array: &PrimitiveArray, ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -91,6 +87,31 @@ impl DeltaData { offset: usize, len: usize, ) -> VortexResult { + Self::validate_parts(&bases, &deltas, offset, len)?; + + // SAFETY: validation done above + Ok(unsafe { Self::new_unchecked(bases, deltas, offset) }) + } + + pub(crate) fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { + Self::validate_parts(self.bases(), self.deltas(), self.offset, len)?; + let expected_dtype = self + .bases() + .dtype() + .with_nullability(self.deltas().dtype().nullability()); + vortex_ensure!( + dtype == &expected_dtype, + "DeltaArray dtype mismatch: expected {expected_dtype}, got {dtype}" + ); + Ok(()) + } + + fn validate_parts( + bases: &ArrayRef, + deltas: &ArrayRef, + offset: usize, + len: usize, + ) -> VortexResult<()> { vortex_ensure!(offset < 1024, "offset must be less than 1024: {offset}"); vortex_ensure!( offset + len <= deltas.len(), @@ -105,8 +126,8 @@ impl DeltaData { ); vortex_ensure!( - bases.dtype().is_int(), - "DeltaArray: dtype must be an integer, got {}", + bases.dtype().is_unsigned_int(), + "DeltaArray: dtype must be an unsigned integer, got {}", bases.dtype() ); @@ -122,24 +143,13 @@ impl DeltaData { "bases length ({}) must be a multiple of LANES ({lanes})", bases.len(), ); - - // SAFETY: validation done above - Ok(unsafe { Self::new_unchecked(bases, deltas, offset, len) }) + Ok(()) } - pub(crate) unsafe fn new_unchecked( - bases: ArrayRef, - deltas: ArrayRef, - offset: usize, - logical_len: usize, - ) -> Self { - let dtype = bases.dtype().with_nullability(deltas.dtype().nullability()); + pub(crate) unsafe fn new_unchecked(bases: ArrayRef, deltas: ArrayRef, offset: usize) -> Self { Self { offset, - len: logical_len, - dtype, slots: vec![Some(bases), Some(deltas)], - stats_set: Default::default(), } } @@ -157,25 +167,6 @@ impl DeltaData { .vortex_expect("DeltaArray deltas slot") } - pub(crate) fn lanes(&self) -> usize { - lane_count(self.dtype().as_ptype()) - } - - #[inline] - pub fn len(&self) -> usize { - self.len - } - - #[inline] - pub fn is_empty(&self) -> bool { - self.len == 0 - } - - #[inline] - pub fn dtype(&self) -> &DType { - &self.dtype - } - #[inline] /// The logical offset into the first chunk of [`Self::deltas`]. pub fn offset(&self) -> usize { @@ -189,10 +180,6 @@ impl DeltaData { pub(crate) fn deltas_len(&self) -> usize { self.deltas().len() } - - pub(crate) fn stats_set(&self) -> &ArrayStats { - &self.stats_set - } } pub(crate) fn lane_count(ptype: PType) -> usize { diff --git a/encodings/fastlanes/src/delta/compute/cast.rs b/encodings/fastlanes/src/delta/compute/cast.rs index 46575c5dc6b..bd1d4c29311 100644 --- a/encodings/fastlanes/src/delta/compute/cast.rs +++ b/encodings/fastlanes/src/delta/compute/cast.rs @@ -11,7 +11,6 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use vortex_error::vortex_panic; -use crate::DeltaData; use crate::delta::Delta; impl CastReduce for Delta { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { @@ -38,7 +37,7 @@ impl CastReduce for Delta { // Create a new DeltaArray with the casted components, preserving offset and logical length Ok(Some( - DeltaData::try_new(casted_bases, casted_deltas, array.offset(), array.len())? + Delta::try_new(casted_bases, casted_deltas, array.offset(), array.len())? .into_array(), )) } @@ -62,16 +61,15 @@ mod tests { use vortex_buffer::buffer; use vortex_session::VortexSession; - use crate::DeltaData; + use crate::Delta; static SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); #[test] fn test_cast_delta_u8_to_u32() { let primitive = PrimitiveArray::from_iter([10u8, 20, 30, 40, 50]); - let array = - DeltaData::try_from_primitive_array(&primitive, &mut SESSION.create_execution_ctx()) - .unwrap(); + let array = Delta::try_from_primitive_array(&primitive, &mut SESSION.create_execution_ctx()) + .unwrap(); let casted = array .into_array() @@ -94,9 +92,8 @@ mod tests { buffer![100u16, 0, 200, 300, 0], vortex_array::validity::Validity::NonNullable, ); - let array = - DeltaData::try_from_primitive_array(&values, &mut SESSION.create_execution_ctx()) - .unwrap(); + let array = Delta::try_from_primitive_array(&values, &mut SESSION.create_execution_ctx()) + .unwrap(); let casted = array .into_array() @@ -135,7 +132,7 @@ mod tests { )] fn test_cast_delta_conformance(#[case] primitive: PrimitiveArray) { let delta_array = - DeltaData::try_from_primitive_array(&primitive, &mut SESSION.create_execution_ctx()) + Delta::try_from_primitive_array(&primitive, &mut SESSION.create_execution_ctx()) .unwrap(); test_cast_conformance(&delta_array.into_array()); } diff --git a/encodings/fastlanes/src/delta/vtable/mod.rs b/encodings/fastlanes/src/delta/vtable/mod.rs index 887de327a82..137c3ff202f 100644 --- a/encodings/fastlanes/src/delta/vtable/mod.rs +++ b/encodings/fastlanes/src/delta/vtable/mod.rs @@ -6,6 +6,7 @@ use std::hash::Hash; use fastlanes::FastLanes; use prost::Message; use vortex_array::Array; +use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; @@ -15,14 +16,12 @@ use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_array::match_each_unsigned_integer_ptype; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::VTable; use vortex_error::VortexResult; @@ -55,29 +54,15 @@ pub struct DeltaMetadata { impl VTable for Delta { type ArrayData = DeltaData; - type Metadata = ProstMetadata; - type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &DeltaData) -> &Self { - &Delta - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &DeltaData) -> usize { - array.len() - } - - fn dtype(array: &DeltaData) -> &DType { - array.dtype() - } - - fn stats(array: &DeltaData) -> &ArrayStats { - array.stats_set() + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate(dtype, len) } fn array_hash(array: &DeltaData, state: &mut H, precision: Precision) { @@ -131,41 +116,34 @@ impl VTable for Delta { Ok(()) } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(DeltaMetadata { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some( + DeltaMetadata { deltas_len: array.deltas().len() as u64, offset: array.offset() as u32, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.0.encode_to_vec())) + } + .encode_to_vec(), + )) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata(DeltaMetadata::decode(bytes)?)) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, - _buffers: &[BufferHandle], + metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { - assert_eq!(children.len(), 2); + vortex_ensure!(buffers.is_empty(), "DeltaArray expects 0 buffers, got {}", buffers.len()); + vortex_ensure!(children.len() == 2, "DeltaArray expects 2 children, got {}", children.len()); + let metadata = DeltaMetadata::decode(metadata)?; let ptype = PType::try_from(dtype)?; let lanes = match_each_unsigned_integer_ptype!(ptype, |T| { ::LANES }); // Compute the length of the bases array - let deltas_len = usize::try_from(metadata.0.deltas_len) - .map_err(|_| vortex_err!("deltas_len {} overflowed usize", metadata.0.deltas_len))?; + let deltas_len = usize::try_from(metadata.deltas_len) + .map_err(|_| vortex_err!("deltas_len {} overflowed usize", metadata.deltas_len))?; let num_chunks = deltas_len / 1024; let remainder_base_size = if deltas_len % 1024 > 0 { 1 } else { 0 }; let bases_len = num_chunks * lanes + remainder_base_size; @@ -173,7 +151,7 @@ impl VTable for Delta { let bases = children.get(0, dtype, bases_len)?; let deltas = children.get(1, dtype, deltas_len)?; - DeltaData::try_new(bases, deltas, metadata.0.offset as usize, len) + DeltaData::try_new(bases, deltas, metadata.offset as usize, len) } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { @@ -189,21 +167,34 @@ pub struct Delta; impl Delta { pub const ID: ArrayId = ArrayId::new_ref("fastlanes.delta"); + pub fn try_new( + bases: ArrayRef, + deltas: ArrayRef, + offset: usize, + len: usize, + ) -> VortexResult { + let dtype = bases.dtype().with_nullability(deltas.dtype().nullability()); + let data = DeltaData::try_new(bases, deltas, offset, len)?; + Array::try_from_parts(ArrayNew::new(Delta, dtype, len, data)) + } + /// Compress a primitive array using Delta encoding. pub fn try_from_primitive_array( array: &PrimitiveArray, ctx: &mut ExecutionCtx, ) -> VortexResult { - Array::try_from_data(DeltaData::try_from_primitive_array(array, ctx)?) + let logical_len = array.len(); + let (bases, deltas) = crate::delta::array::delta_compress::delta_compress(array, ctx)?; + Self::try_new(bases.into_array(), deltas.into_array(), 0, logical_len) } } #[cfg(test)] mod tests { + use vortex_array::ProstMetadata; use vortex_array::test_harness::check_metadata; use super::DeltaMetadata; - use super::ProstMetadata; #[cfg_attr(miri, ignore)] #[test] diff --git a/encodings/fastlanes/src/delta/vtable/operations.rs b/encodings/fastlanes/src/delta/vtable/operations.rs index cae9b701ee9..731588a67bb 100644 --- a/encodings/fastlanes/src/delta/vtable/operations.rs +++ b/encodings/fastlanes/src/delta/vtable/operations.rs @@ -38,18 +38,15 @@ mod tests { use vortex_error::VortexExpect; use vortex_session::VortexSession; + use crate::Delta; use crate::DeltaArray; - use crate::DeltaData; static SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); fn da(array: &PrimitiveArray) -> DeltaArray { - DeltaArray::try_from_data( - DeltaData::try_from_primitive_array(array, &mut SESSION.create_execution_ctx()) - .unwrap(), - ) - .vortex_expect("DeltaData is always valid") + Delta::try_from_primitive_array(array, &mut SESSION.create_execution_ctx()) + .vortex_expect("Delta array construction should succeed") } #[test] diff --git a/encodings/fastlanes/src/delta/vtable/slice.rs b/encodings/fastlanes/src/delta/vtable/slice.rs index 3068569d3c0..d479a5ab288 100644 --- a/encodings/fastlanes/src/delta/vtable/slice.rs +++ b/encodings/fastlanes/src/delta/vtable/slice.rs @@ -10,7 +10,7 @@ use vortex_array::IntoArray; use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; -use crate::DeltaData; +use crate::delta::array::lane_count; use crate::delta::vtable::Delta; impl SliceReduce for Delta { @@ -23,7 +23,7 @@ impl SliceReduce for Delta { let bases = array.bases(); let deltas = array.deltas(); - let lanes = array.lanes(); + let lanes = lane_count(array.dtype().as_ptype()); let new_bases = bases.slice( min(start_chunk * lanes, array.bases_len())..min(stop_chunk * lanes, array.bases_len()), @@ -33,10 +33,9 @@ impl SliceReduce for Delta { min(start_chunk * 1024, array.deltas_len())..min(stop_chunk * 1024, array.deltas_len()), )?; - // SAFETY: slicing valid bases/deltas preserves correctness - Ok(Some(unsafe { - DeltaData::new_unchecked(new_bases, new_deltas, physical_start % 1024, range.len()) - .into_array() - })) + Ok(Some( + Delta::try_new(new_bases, new_deltas, physical_start % 1024, range.len())? + .into_array(), + )) } } diff --git a/encodings/fastlanes/src/for/array/for_compress.rs b/encodings/fastlanes/src/for/array/for_compress.rs index b911e20fa17..8bf8a8cd314 100644 --- a/encodings/fastlanes/src/for/array/for_compress.rs +++ b/encodings/fastlanes/src/for/array/for_compress.rs @@ -5,20 +5,18 @@ use num_traits::PrimInt; use num_traits::WrappingSub; use vortex_array::IntoArray; use vortex_array::arrays::PrimitiveArray; -use vortex_array::arrays::primitive::PrimitiveData; use vortex_array::dtype::NativePType; use vortex_array::expr::stats::Stat; use vortex_array::match_each_integer_ptype; -use vortex_array::stats::ArrayStats; use vortex_error::VortexResult; use vortex_error::vortex_err; +use crate::FoR; use crate::FoRArray; use crate::FoRData; impl FoRData { pub fn encode(array: PrimitiveArray) -> VortexResult { let array_ref = array.clone().into_array(); - let stats = ArrayStats::from(array_ref.statistics().to_owned()); let min = array_ref .statistics() .compute_stat(Stat::Min)? @@ -27,13 +25,8 @@ impl FoRData { let encoded = match_each_integer_ptype!(array.ptype(), |T| { compress_primitive::(array, T::try_from(&min)?)?.into_array() }); - let for_data = FoRData::try_new(encoded, min)?; - let for_array = FoRArray::try_from_data(for_data)?; - let for_ref = for_array.clone().into_array(); - for_array - .stats_set() - .to_ref(&for_ref) - .inherit_from(stats.to_ref(&for_ref)); + let for_array = FoR::try_new(encoded, min)?; + for_array.statistics().inherit_from(array_ref.statistics()); Ok(for_array) } } @@ -41,10 +34,10 @@ impl FoRData { fn compress_primitive( parray: PrimitiveArray, min: T, -) -> VortexResult { +) -> VortexResult { // Set null values to the min value, ensuring that decompress into a value in the primitive // range (and stop them wrapping around). - parray + let encoded = parray .into_data() .map_each_with_validity::(|(v, bool)| { if bool { @@ -52,7 +45,12 @@ fn compress_primitive( } else { T::zero() } - }) + })?; + Ok(PrimitiveArray::from_buffer_handle( + encoded.buffer_handle().clone(), + encoded.ptype(), + encoded.validity(), + )) } #[cfg(test)] @@ -137,7 +135,7 @@ mod test { let expect = PrimitiveArray::from_iter((0u32..1024).map(|x| x % 7 + 10)); let array = PrimitiveArray::from_iter((0u32..1024).map(|x| x % 7)); let bp = BitPackedData::encode(&array.into_array(), 3).unwrap(); - let compressed = FoRData::try_new(bp.into_array(), 10u32.into()).unwrap(); + let compressed = FoR::try_new(bp.into_array(), 10u32.into()).unwrap(); assert_arrays_eq!(compressed, expect); } @@ -147,9 +145,7 @@ mod test { let expect = PrimitiveArray::from_iter((0u32..1024).map(|x| x % 7 + 10)); let array = PrimitiveArray::from_iter((0u32..1024).map(|x| x % 7)); let bp = BitPackedData::encode(&array.into_array(), 2).unwrap(); - let compressed = FoRArray::try_from_data( - FoRData::try_new(bp.clone().into_array(), 10u32.into()).unwrap(), - )?; + let compressed = FoR::try_new(bp.clone().into_array(), 10u32.into())?; let decompressed = fused_decompress::( &compressed, bp.as_view(), diff --git a/encodings/fastlanes/src/for/array/for_decompress.rs b/encodings/fastlanes/src/for/array/for_decompress.rs index 13de40e6365..639d369a1fc 100644 --- a/encodings/fastlanes/src/for/array/for_decompress.rs +++ b/encodings/fastlanes/src/for/array/for_decompress.rs @@ -107,7 +107,7 @@ pub(crate) fn fused_decompress< let mut uninit_range = builder.uninit_range(bp.len()); unsafe { // Append a dense null Mask. - uninit_range.append_mask(bp.validity_mask()); + uninit_range.append_mask(bp.validity_mask(bp.len(), bp.dtype().nullability())); } // SAFETY: `decode_into` will initialize all values in this range. @@ -116,7 +116,7 @@ pub(crate) fn fused_decompress< // Decode all chunks (initial, full, and trailer) in one call. unpacked.decode_into(uninit_slice); - if let Some(ref patches) = bp.patches() { + if let Some(ref patches) = bp.patches(bp.len()) { bitpack_decompress::apply_patches_to_uninit_range_fn( &mut uninit_range, patches, diff --git a/encodings/fastlanes/src/for/array/mod.rs b/encodings/fastlanes/src/for/array/mod.rs index 74d88580bd3..42b3b4a5230 100644 --- a/encodings/fastlanes/src/for/array/mod.rs +++ b/encodings/fastlanes/src/for/array/mod.rs @@ -5,10 +5,9 @@ use vortex_array::ArrayRef; use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; -use vortex_array::stats::ArrayStats; use vortex_error::VortexExpect as _; use vortex_error::VortexResult; -use vortex_error::vortex_bail; +use vortex_error::vortex_ensure; pub mod for_compress; pub mod for_decompress; @@ -26,24 +25,15 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["encoded"]; pub struct FoRData { pub(super) slots: Vec>, pub(super) reference: Scalar, - pub(super) stats_set: ArrayStats, } impl FoRData { - pub fn try_new(encoded: ArrayRef, reference: Scalar) -> VortexResult { - if reference.is_null() { - vortex_bail!("Reference value cannot be null"); - } - let reference = reference.cast( - &reference - .dtype() - .with_nullability(encoded.dtype().nullability()), - )?; + pub(crate) fn try_new(encoded: ArrayRef, reference: Scalar) -> VortexResult { + Self::validate_parts(&encoded, &reference, reference.dtype(), encoded.len())?; Ok(Self { slots: vec![Some(encoded)], reference, - stats_set: Default::default(), }) } @@ -51,10 +41,42 @@ impl FoRData { Self { slots: vec![Some(encoded)], reference, - stats_set: Default::default(), } } + pub(crate) fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { + Self::validate_parts(self.encoded(), &self.reference, dtype, len) + } + + fn validate_parts( + encoded: &ArrayRef, + reference: &Scalar, + dtype: &DType, + len: usize, + ) -> VortexResult<()> { + vortex_ensure!( + !reference.is_null(), + "Reference value cannot be null" + ); + vortex_ensure!(dtype.is_int(), "FoR requires an integer dtype, got {dtype}"); + vortex_ensure!( + reference.dtype() == dtype, + "FoR reference dtype mismatch: expected {dtype}, got {}", + reference.dtype() + ); + vortex_ensure!( + encoded.dtype() == dtype, + "FoR encoded dtype mismatch: expected {dtype}, got {}", + encoded.dtype() + ); + vortex_ensure!( + encoded.len() == len, + "FoR encoded length mismatch: expected {len}, got {}", + encoded.len() + ); + Ok(()) + } + /// Returns the length of the array. #[inline] pub fn len(&self) -> usize { @@ -89,8 +111,4 @@ impl FoRData { pub fn reference_scalar(&self) -> &Scalar { &self.reference } - - pub(crate) fn stats_set(&self) -> &ArrayStats { - &self.stats_set - } } diff --git a/encodings/fastlanes/src/for/compute/cast.rs b/encodings/fastlanes/src/for/compute/cast.rs index 884a0a8b94f..e990a012d5d 100644 --- a/encodings/fastlanes/src/for/compute/cast.rs +++ b/encodings/fastlanes/src/for/compute/cast.rs @@ -9,7 +9,6 @@ use vortex_array::dtype::DType; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; -use crate::FoRData; use crate::r#for::FoR; impl CastReduce for FoR { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { @@ -22,9 +21,7 @@ impl CastReduce for FoR { let casted_child = array.encoded().cast(dtype.clone())?; let casted_reference = array.reference_scalar().cast(dtype)?; - Ok(Some( - FoRData::try_new(casted_child, casted_reference)?.into_array(), - )) + Ok(Some(FoR::try_new(casted_child, casted_reference)?.into_array())) } } @@ -44,12 +41,11 @@ mod tests { use vortex_buffer::buffer; use vortex_error::VortexExpect; + use crate::FoR; use crate::FoRArray; - use crate::FoRData; fn for_arr(encoded: ArrayRef, reference: Scalar) -> FoRArray { - FoRArray::try_from_data(FoRData::try_new(encoded, reference).unwrap()) - .vortex_expect("FoRData is always valid") + FoR::try_new(encoded, reference).vortex_expect("FoR array construction should succeed") } #[test] diff --git a/encodings/fastlanes/src/for/compute/compare.rs b/encodings/fastlanes/src/for/compute/compare.rs index f23ec8c0256..93d25c7c846 100644 --- a/encodings/fastlanes/src/for/compute/compare.rs +++ b/encodings/fastlanes/src/for/compute/compare.rs @@ -23,7 +23,6 @@ use vortex_error::VortexExpect as _; use vortex_error::VortexResult; use crate::FoR; -use crate::FoRData; impl CompareKernel for FoR { fn compare( @@ -37,7 +36,7 @@ impl CompareKernel for FoR { { match_each_integer_ptype!(constant.ptype(), |T| { return compare_constant( - &lhs, + lhs, constant .typed_value::() .vortex_expect("null scalar handled in adaptor"), @@ -52,7 +51,7 @@ impl CompareKernel for FoR { } fn compare_constant( - lhs: &FoRData, + lhs: ArrayView<'_, FoR>, mut rhs: T, nullability: Nullability, operator: CompareOperator, @@ -81,10 +80,7 @@ where let rhs = Scalar::primitive(rhs, nullability); lhs.encoded() - .binary( - ConstantArray::new(rhs, lhs.len()).into_array(), - Operator::from(operator), - ) + .binary(ConstantArray::new(rhs, lhs.len()).into_array(), Operator::from(operator)) .map(Some) } @@ -99,12 +95,11 @@ mod tests { use vortex_buffer::buffer; use super::*; + use crate::FoR; use crate::FoRArray; - use crate::FoRData; fn for_arr(encoded: ArrayRef, reference: Scalar) -> FoRArray { - FoRArray::try_from_data(FoRData::try_new(encoded, reference).unwrap()) - .vortex_expect("FoRData is always valid") + FoR::try_new(encoded, reference).vortex_expect("FoR array construction should succeed") } #[test] @@ -116,13 +111,18 @@ mod tests { reference, ); - let result = compare_constant(&lhs, 30i32, Nullability::NonNullable, CompareOperator::Eq) + let result = compare_constant( + lhs.as_view(), + 30i32, + Nullability::NonNullable, + CompareOperator::Eq, + ) .unwrap() .unwrap(); assert_arrays_eq!(result, BoolArray::from_iter([false, true, false].map(Some))); let result = compare_constant( - &lhs, + lhs.as_view(), 12i32, Nullability::NonNullable, CompareOperator::NotEq, @@ -138,7 +138,7 @@ mod tests { CompareOperator::Gte, ] { assert!( - compare_constant(&lhs, 30i32, Nullability::NonNullable, op) + compare_constant(lhs.as_view(), 30i32, Nullability::NonNullable, op) .unwrap() .is_none() ); @@ -155,14 +155,24 @@ mod tests { ); assert_eq!( - compare_constant(&lhs, 30i32, Nullability::Nullable, CompareOperator::Eq) + compare_constant( + lhs.as_view(), + 30i32, + Nullability::Nullable, + CompareOperator::Eq, + ) .unwrap() .unwrap() .dtype(), &DType::Bool(Nullability::Nullable) ); assert_eq!( - compare_constant(&lhs, 30i32, Nullability::NonNullable, CompareOperator::Eq) + compare_constant( + lhs.as_view(), + 30i32, + Nullability::NonNullable, + CompareOperator::Eq, + ) .unwrap() .unwrap() .dtype(), @@ -179,7 +189,12 @@ mod tests { reference, ); - let result = compare_constant(&lhs, -1i32, Nullability::NonNullable, CompareOperator::Eq) + let result = compare_constant( + lhs.as_view(), + -1i32, + Nullability::NonNullable, + CompareOperator::Eq, + ) .unwrap() .unwrap(); assert_arrays_eq!( @@ -188,7 +203,7 @@ mod tests { ); let result = compare_constant( - &lhs, + lhs.as_view(), -1i32, Nullability::NonNullable, CompareOperator::NotEq, @@ -212,7 +227,7 @@ mod tests { ); let result = compare_constant( - &lhs, + lhs.as_view(), 435090932899640449i64, Nullability::Nullable, CompareOperator::Eq, @@ -222,7 +237,7 @@ mod tests { assert_arrays_eq!(result, BoolArray::from_iter([Some(false), Some(true)])); let result = compare_constant( - &lhs, + lhs.as_view(), 435090932899640449i64, Nullability::Nullable, CompareOperator::NotEq, diff --git a/encodings/fastlanes/src/for/compute/is_sorted.rs b/encodings/fastlanes/src/for/compute/is_sorted.rs index 05d5e31836e..95504cdb7c7 100644 --- a/encodings/fastlanes/src/for/compute/is_sorted.rs +++ b/encodings/fastlanes/src/for/compute/is_sorted.rs @@ -10,6 +10,7 @@ use vortex_array::aggregate_fn::fns::is_sorted::IsSorted; use vortex_array::aggregate_fn::fns::is_sorted::is_sorted; use vortex_array::aggregate_fn::fns::is_sorted::is_strict_sorted; use vortex_array::aggregate_fn::kernels::DynAggregateKernel; +use vortex_array::arrays::PrimitiveArray; use vortex_array::scalar::Scalar; use vortex_error::VortexResult; @@ -34,9 +35,12 @@ impl DynAggregateKernel for FoRIsSortedKernel { }; let encoded = array.encoded().to_primitive(); - let unsigned_array = encoded - .reinterpret_cast(encoded.ptype().to_unsigned()) - .into_array(); + let unsigned_array = PrimitiveArray::from_buffer_handle( + encoded.buffer_handle().clone(), + encoded.ptype().to_unsigned(), + encoded.validity(), + ) + .into_array(); let result = if options.strict { is_strict_sorted(&unsigned_array, ctx)? diff --git a/encodings/fastlanes/src/for/compute/mod.rs b/encodings/fastlanes/src/for/compute/mod.rs index 7704dc02ff5..14f990e3ad6 100644 --- a/encodings/fastlanes/src/for/compute/mod.rs +++ b/encodings/fastlanes/src/for/compute/mod.rs @@ -16,7 +16,6 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::FoR; -use crate::FoRData; impl TakeExecute for FoR { fn take( @@ -25,22 +24,16 @@ impl TakeExecute for FoR { _ctx: &mut ExecutionCtx, ) -> VortexResult> { Ok(Some( - FoRData::try_new( - array.encoded().take(indices.clone())?, - array.reference_scalar().clone(), - )? - .into_array(), + FoR::try_new(array.encoded().take(indices.clone())?, array.reference_scalar().clone())? + .into_array(), )) } } impl FilterReduce for FoR { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { - FoRData::try_new( - array.encoded().filter(mask.clone())?, - array.reference_scalar().clone(), - ) - .map(|a| Some(a.into_array())) + FoR::try_new(array.encoded().filter(mask.clone())?, array.reference_scalar().clone()) + .map(|a| Some(a.into_array())) } } @@ -56,11 +49,10 @@ mod test { use vortex_error::VortexExpect; use crate::FoRArray; - use crate::FoRData; + use crate::FoR; fn fa(encoded: ArrayRef, reference: Scalar) -> FoRArray { - FoRArray::try_from_data(FoRData::try_new(encoded, reference).unwrap()) - .vortex_expect("FoRData is always valid") + FoR::try_new(encoded, reference).vortex_expect("FoR array construction should succeed") } #[test] @@ -111,11 +103,10 @@ mod tests { use vortex_error::VortexExpect; use crate::FoRArray; - use crate::FoRData; + use crate::FoR; fn fa(encoded: ArrayRef, reference: Scalar) -> FoRArray { - FoRArray::try_from_data(FoRData::try_new(encoded, reference).unwrap()) - .vortex_expect("FoRData is always valid") + FoR::try_new(encoded, reference).vortex_expect("FoR array construction should succeed") } #[rstest] diff --git a/encodings/fastlanes/src/for/vtable/mod.rs b/encodings/fastlanes/src/for/vtable/mod.rs index 80a2b165b91..2af970af65b 100644 --- a/encodings/fastlanes/src/for/vtable/mod.rs +++ b/encodings/fastlanes/src/for/vtable/mod.rs @@ -5,6 +5,7 @@ use std::fmt::Debug; use std::hash::Hash; use vortex_array::Array; +use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; @@ -20,7 +21,6 @@ use vortex_array::dtype::DType; use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityVTableFromChild; @@ -48,29 +48,15 @@ vtable!(FoR, FoR, FoRData); impl VTable for FoR { type ArrayData = FoRData; - type Metadata = Scalar; - type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; - fn vtable(_array: &FoRData) -> &Self { - &FoR - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &FoRData) -> usize { - array.encoded().len() - } - - fn dtype(array: &FoRData) -> &DType { - array.reference_scalar().dtype() - } - - fn stats(array: &FoRData) -> &ArrayStats { - array.stats_set() + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate(dtype, len) } fn array_hash(array: &FoRData, state: &mut H, precision: Precision) { @@ -114,33 +100,23 @@ impl VTable for FoR { Ok(()) } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(array.reference_scalar().clone()) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { // Note that we **only** serialize the optional scalar value (not including the dtype). - Ok(Some(ScalarValue::to_proto_bytes(metadata.value()))) + Ok(Some(ScalarValue::to_proto_bytes( + array.reference_scalar().value(), + ))) } fn deserialize( - bytes: &[u8], - dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - session: &VortexSession, - ) -> VortexResult { - let scalar_value = ScalarValue::from_proto_bytes(bytes, dtype, session)?; - Scalar::try_new(dtype.clone(), scalar_value) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, - _buffers: &[BufferHandle], + metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + session: &VortexSession, ) -> VortexResult { + vortex_ensure!(buffers.is_empty(), "FoRArray expects 0 buffers, got {}", buffers.len()); if children.len() != 1 { vortex_bail!( "Expected 1 child for FoR encoding, found {}", @@ -148,9 +124,11 @@ impl VTable for FoR { ) } + let scalar_value = ScalarValue::from_proto_bytes(metadata, dtype, session)?; + let reference = Scalar::try_new(dtype.clone(), scalar_value)?; let encoded = children.get(0, dtype, len)?; - FoRData::try_new(encoded, metadata.clone()) + FoRData::try_new(encoded, reference) } fn reduce_parent( @@ -183,7 +161,14 @@ impl FoR { /// Construct a new FoR array from an encoded array and a reference scalar. pub fn try_new(encoded: ArrayRef, reference: Scalar) -> VortexResult { - Array::try_from_data(FoRData::try_new(encoded, reference)?) + vortex_ensure!(!reference.is_null(), "Reference value cannot be null"); + let dtype = reference + .dtype() + .with_nullability(encoded.dtype().nullability()); + let reference = reference.cast(&dtype)?; + let len = encoded.len(); + let data = FoRData::try_new(encoded, reference)?; + Array::try_from_parts(ArrayNew::new(FoR, dtype, len, data)) } /// Encode a primitive array using Frame of Reference encoding. diff --git a/encodings/fastlanes/src/for/vtable/rules.rs b/encodings/fastlanes/src/for/vtable/rules.rs index 38d4b4186f2..3f839fe74dd 100644 --- a/encodings/fastlanes/src/for/vtable/rules.rs +++ b/encodings/fastlanes/src/for/vtable/rules.rs @@ -13,7 +13,6 @@ use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; use vortex_error::VortexResult; use crate::FoR; -use crate::FoRData; pub(super) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ // TODO: add BetweenReduceAdaptor(FoR) @@ -35,12 +34,12 @@ impl ArrayParentReduceRule for FoRFilterPushDownRule { parent: ArrayView<'_, Filter>, _child_idx: usize, ) -> VortexResult> { - let new_array = unsafe { - FoRData::new_unchecked( + Ok(Some( + FoR::try_new( child.encoded().filter(parent.filter_mask().clone())?, - child.reference.clone(), - ) - }; - Ok(Some(new_array.into_array())) + child.reference_scalar().clone(), + )? + .into_array(), + )) } } diff --git a/encodings/fastlanes/src/for/vtable/slice.rs b/encodings/fastlanes/src/for/vtable/slice.rs index c7571ac274a..58914fad721 100644 --- a/encodings/fastlanes/src/for/vtable/slice.rs +++ b/encodings/fastlanes/src/for/vtable/slice.rs @@ -10,17 +10,12 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::FoR; -use crate::FoRData; impl SliceReduce for FoR { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - // SAFETY: Just slicing encoded data does not affect FOR. - Ok(Some(unsafe { - FoRData::new_unchecked( - array.encoded().slice(range)?, - array.reference_scalar().clone(), - ) - .into_array() - })) + Ok(Some( + FoR::try_new(array.encoded().slice(range)?, array.reference_scalar().clone())? + .into_array(), + )) } } diff --git a/encodings/fastlanes/src/rle/array/mod.rs b/encodings/fastlanes/src/rle/array/mod.rs index d0d207c1795..f9a292e83de 100644 --- a/encodings/fastlanes/src/rle/array/mod.rs +++ b/encodings/fastlanes/src/rle/array/mod.rs @@ -3,8 +3,8 @@ use vortex_array::ArrayRef; use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; -use vortex_array::stats::ArrayStats; use vortex_error::VortexExpect as _; use vortex_error::VortexResult; use vortex_error::vortex_ensure; @@ -33,12 +33,9 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["values", "indices", "values_i #[derive(Clone, Debug)] pub struct RLEData { - pub(super) dtype: DType, pub(super) slots: Vec>, - pub(super) stats_set: ArrayStats, // Offset relative to the start of the chunk. pub(super) offset: usize, - pub(super) length: usize, } impl RLEData { @@ -47,6 +44,37 @@ impl RLEData { indices: &ArrayRef, value_idx_offsets: &ArrayRef, offset: usize, + length: usize, + ) -> VortexResult<()> { + Self::validate_parts(values, indices, value_idx_offsets, offset, length)?; + Ok(()) + } + + pub(crate) fn validate_against_outer(&self, dtype: &DType, length: usize) -> VortexResult<()> { + Self::validate_parts( + self.values(), + self.indices(), + self.values_idx_offsets(), + self.offset, + length, + )?; + let expected_dtype = DType::Primitive( + self.values().dtype().as_ptype(), + self.indices().dtype().nullability(), + ); + vortex_ensure!( + dtype == &expected_dtype, + "RLE dtype mismatch: expected {expected_dtype}, got {dtype}" + ); + Ok(()) + } + + fn validate_parts( + values: &ArrayRef, + indices: &ArrayRef, + value_idx_offsets: &ArrayRef, + offset: usize, + length: usize, ) -> VortexResult<()> { vortex_ensure!( offset < 1024, @@ -55,8 +83,8 @@ impl RLEData { ); vortex_ensure!( - values.dtype().is_primitive(), - "RLE values must be a primitive type, got {}", + matches!(values.dtype(), DType::Primitive(_, Nullability::NonNullable)), + "RLE values must be a non-nullable primitive type, got {}", values.dtype() ); @@ -71,6 +99,16 @@ impl RLEData { "RLE value idx offsets must be non-nullable unsigned integer, got {}", value_idx_offsets.dtype() ); + vortex_ensure!( + indices.len().is_multiple_of(FL_CHUNK_SIZE), + "RLE indices length must be a multiple of {FL_CHUNK_SIZE}, got {}", + indices.len() + ); + vortex_ensure!( + offset + length <= indices.len(), + "RLE offset + length, {offset} + {length}, must not exceed the indices length {}", + indices.len() + ); vortex_ensure!( indices.len().div_ceil(FL_CHUNK_SIZE) == value_idx_offsets.len(), @@ -104,18 +142,11 @@ impl RLEData { offset: usize, length: usize, ) -> VortexResult { - assert_eq!(indices.len() % FL_CHUNK_SIZE, 0); - Self::validate(&values, &indices, &values_idx_offsets, offset)?; - - // Ensure that the DType has the same nullability as the indices array. - let dtype = DType::Primitive(values.dtype().as_ptype(), indices.dtype().nullability()); + Self::validate(&values, &indices, &values_idx_offsets, offset, length)?; Ok(Self { - dtype, slots: vec![Some(values), Some(indices), Some(values_idx_offsets)], - stats_set: ArrayStats::default(), offset, - length, }) } @@ -124,42 +155,20 @@ impl RLEData { /// # Safety /// The caller must ensure that: /// - `offset + length` does not exceed the length of the indices array - /// - The `dtype` is consistent with the values array's primitive type and validity nullability /// - The `indices` array contains valid indices into chunks of the `values` array /// - The `values_idx_offsets` array contains valid chunk start offsets - /// - The `validity` array has the same length as `length` pub unsafe fn new_unchecked( values: ArrayRef, indices: ArrayRef, values_idx_offsets: ArrayRef, - dtype: DType, offset: usize, - length: usize, ) -> Self { Self { - dtype, slots: vec![Some(values), Some(indices), Some(values_idx_offsets)], - stats_set: ArrayStats::default(), offset, - length, } } - #[inline] - pub fn len(&self) -> usize { - self.length - } - - #[inline] - pub fn is_empty(&self) -> bool { - self.length == 0 - } - - #[inline] - pub fn dtype(&self) -> &DType { - &self.dtype - } - #[inline] pub fn values(&self) -> &ArrayRef { self.slots[VALUES_SLOT] @@ -211,10 +220,6 @@ impl RLEData { pub fn offset(&self) -> usize { self.offset } - - pub(crate) fn stats_set(&self) -> &ArrayStats { - &self.stats_set - } } #[cfg(test)] diff --git a/encodings/fastlanes/src/rle/array/rle_compress.rs b/encodings/fastlanes/src/rle/array/rle_compress.rs index a432097b04e..74554d9deef 100644 --- a/encodings/fastlanes/src/rle/array/rle_compress.rs +++ b/encodings/fastlanes/src/rle/array/rle_compress.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use arrayref::array_mut_ref; -use fastlanes::RLE; +use fastlanes::RLE as FastLanesRLE; use vortex_array::IntoArray; use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; @@ -15,6 +15,7 @@ use vortex_buffer::BufferMut; use vortex_error::VortexResult; use crate::FL_CHUNK_SIZE; +use crate::RLE; use crate::RLEArray; use crate::RLEData; use crate::fill_forward_nulls; @@ -31,8 +32,8 @@ impl RLEData { /// In case the input array length is % 1024 != 0, the last chunk is padded. fn rle_encode_typed(array: &PrimitiveArray) -> VortexResult where - T: NativePType + RLE, - NativeValue: RLE, + T: NativePType + FastLanesRLE, + NativeValue: FastLanesRLE, { // Fill-forward null values so the RLE encoder doesn't see garbage at null positions, // which would create spurious run boundaries and inflate the dictionary. @@ -98,13 +99,13 @@ where // SAFETY: NativeValue is repr(transparent) to T. let values_buf = unsafe { values_buf.transmute::().freeze() }; - RLEArray::try_from_data(RLEData::try_new( + RLE::try_new( values_buf.into_array(), PrimitiveArray::new(indices_buf.freeze(), padded_validity(array)).into_array(), values_idx_offsets.into_array(), 0, array.len(), - )?) + ) } /// Returns validity padded to the next 1024 chunk for a given array. diff --git a/encodings/fastlanes/src/rle/compute/cast.rs b/encodings/fastlanes/src/rle/compute/cast.rs index 88078862816..0d4777a8f9e 100644 --- a/encodings/fastlanes/src/rle/compute/cast.rs +++ b/encodings/fastlanes/src/rle/compute/cast.rs @@ -3,12 +3,12 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; +use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; -use crate::RLEData; use crate::rle::RLE; impl CastReduce for RLE { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { @@ -25,17 +25,16 @@ impl CastReduce for RLE { array.indices().clone() }; - Ok(Some(unsafe { - RLEData::new_unchecked( + Ok(Some( + RLE::try_new( casted_values, casted_indices, array.values_idx_offsets().clone(), - dtype.clone(), array.offset(), array.len(), - ) - .into() - })) + )? + .into_array(), + )) } } diff --git a/encodings/fastlanes/src/rle/kernel.rs b/encodings/fastlanes/src/rle/kernel.rs index 1da88206913..65f8bfdb907 100644 --- a/encodings/fastlanes/src/rle/kernel.rs +++ b/encodings/fastlanes/src/rle/kernel.rs @@ -14,7 +14,6 @@ use vortex_error::VortexResult; use crate::FL_CHUNK_SIZE; use crate::RLE; -use crate::RLEData; pub(crate) static PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ParentKernelSet::lift(&SliceExecuteAdaptor(RLE))]); @@ -46,18 +45,16 @@ impl SliceKernel for RLE { .indices() .slice(chunk_start_idx * FL_CHUNK_SIZE..chunk_end_idx * FL_CHUNK_SIZE)?; - // SAFETY: Slicing preserves all invariants. - Ok(Some(unsafe { - RLEData::new_unchecked( + Ok(Some( + RLE::try_new( sliced_values, sliced_indices, sliced_values_idx_offsets, - array.dtype().clone(), // Keep the offset relative to the first chunk. (array.offset() + range.start) % FL_CHUNK_SIZE, range.len(), - ) - .into_array() - })) + )? + .into_array(), + )) } } diff --git a/encodings/fastlanes/src/rle/vtable/mod.rs b/encodings/fastlanes/src/rle/vtable/mod.rs index 63eeb7b5706..b449facaa52 100644 --- a/encodings/fastlanes/src/rle/vtable/mod.rs +++ b/encodings/fastlanes/src/rle/vtable/mod.rs @@ -5,6 +5,7 @@ use std::hash::Hash; use prost::Message; use vortex_array::Array; +use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; @@ -14,17 +15,14 @@ use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::VTable; -use vortex_array::vtable::ValidityVTableFromChildSliceHelper; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; @@ -61,29 +59,15 @@ pub struct RLEMetadata { impl VTable for RLE { type ArrayData = RLEData; - type Metadata = ProstMetadata; - type OperationsVTable = Self; - type ValidityVTable = ValidityVTableFromChildSliceHelper; - - fn vtable(_array: &RLEData) -> &Self { - &RLE - } + type ValidityVTable = Self; fn id(&self) -> ArrayId { Self::ID } - fn len(array: &RLEData) -> usize { - array.len() - } - - fn dtype(array: &RLEData) -> &DType { - array.dtype() - } - - fn stats(array: &RLEData) -> &ArrayStats { - array.stats_set() + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate_against_outer(dtype, len) } fn array_hash(array: &RLEData, state: &mut H, precision: Precision) { @@ -141,39 +125,31 @@ impl VTable for RLE { Ok(()) } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(RLEMetadata { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some( + RLEMetadata { values_len: array.values().len() as u64, indices_len: array.indices().len() as u64, indices_ptype: PType::try_from(array.indices().dtype())? as i32, values_idx_offsets_len: array.values_idx_offsets().len() as u64, values_idx_offsets_ptype: PType::try_from(array.values_idx_offsets().dtype())? as i32, offset: array.offset() as u64, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.0.encode_to_vec())) + } + .encode_to_vec(), + )) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata(RLEMetadata::decode(bytes)?)) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, - _buffers: &[BufferHandle], + metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { - let metadata = &metadata.0; + vortex_ensure!(buffers.is_empty(), "RLEArray expects 0 buffers, got {}", buffers.len()); + let metadata = RLEMetadata::decode(metadata)?; let values = children.get( 0, &DType::Primitive(dtype.as_ptype(), Nullability::NonNullable), @@ -226,6 +202,18 @@ pub struct RLE; impl RLE { pub const ID: ArrayId = ArrayId::new_ref("fastlanes.rle"); + pub fn try_new( + values: ArrayRef, + indices: ArrayRef, + values_idx_offsets: ArrayRef, + offset: usize, + length: usize, + ) -> VortexResult { + let dtype = DType::Primitive(values.dtype().as_ptype(), indices.dtype().nullability()); + let data = RLEData::try_new(values, indices, values_idx_offsets, offset, length)?; + Array::try_from_parts(ArrayNew::new(RLE, dtype, length, data)) + } + /// Create a new RLE array without validation. /// /// # Safety @@ -234,14 +222,13 @@ impl RLE { values: ArrayRef, indices: ArrayRef, values_idx_offsets: ArrayRef, - dtype: DType, offset: usize, length: usize, ) -> RLEArray { - Array::try_from_data(unsafe { - RLEData::new_unchecked(values, indices, values_idx_offsets, dtype, offset, length) - }) - .vortex_expect("RLEData is always valid") + let dtype = DType::Primitive(values.dtype().as_ptype(), indices.dtype().nullability()); + let data = unsafe { RLEData::new_unchecked(values, indices, values_idx_offsets, offset) }; + Array::try_from_parts(ArrayNew::new(RLE, dtype, length, data)) + .vortex_expect("pre-validated RLE parts must be valid") } /// Encode a primitive array using FastLanes RLE. @@ -252,9 +239,9 @@ impl RLE { #[cfg(test)] mod tests { + use vortex_array::ProstMetadata; use vortex_array::test_harness::check_metadata; - use super::ProstMetadata; use super::RLEMetadata; #[cfg_attr(miri, ignore)] diff --git a/encodings/fastlanes/src/rle/vtable/validity.rs b/encodings/fastlanes/src/rle/vtable/validity.rs index f46998774cd..ecef8047cb1 100644 --- a/encodings/fastlanes/src/rle/vtable/validity.rs +++ b/encodings/fastlanes/src/rle/vtable/validity.rs @@ -1,22 +1,16 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use vortex_array::ArrayRef; -use vortex_array::vtable::ValidityChild; -use vortex_array::vtable::ValidityChildSliceHelper; +use vortex_array::ArrayView; +use vortex_array::vtable::ValidityVTable; +use vortex_error::VortexResult; -use super::RLE; -use crate::RLEData; +use crate::rle::RLE; -impl ValidityChild for RLE { - fn validity_child(array: &RLEData) -> &ArrayRef { - array.indices() - } -} - -impl ValidityChildSliceHelper for RLEData { - fn unsliced_child_and_slice(&self) -> (&ArrayRef, usize, usize) { - let (start, len) = (self.offset(), self.len()); - (self.indices(), start, start + len) +impl ValidityVTable for RLE { + fn validity(array: ArrayView<'_, RLE>) -> VortexResult { + let start = array.offset(); + let stop = start + array.len(); + array.indices().slice(start..stop)?.validity() } } diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index b98d5d9edd6..a93ef931e11 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -9,20 +9,19 @@ use std::sync::LazyLock; use fsst::Compressor; use fsst::Decompressor; use fsst::Symbol; +use prost::Message as _; use vortex_array::Array; +use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::Canonical; -use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; -use vortex_array::SerializeMetadata; use vortex_array::arrays::VarBin; use vortex_array::arrays::VarBinArray; use vortex_array::buffer::BufferHandle; @@ -32,7 +31,6 @@ use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; @@ -74,29 +72,15 @@ impl FSSTMetadata { impl VTable for FSST { type ArrayData = FSSTData; - - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; - fn vtable(_array: &Self::ArrayData) -> &Self { - &FSST - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &FSSTData) -> usize { - array.codes().len() - } - - fn dtype(array: &FSSTData) -> &DType { - &array.dtype - } - - fn stats(array: &FSSTData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate(dtype, len) } fn array_hash(array: &FSSTData, state: &mut H, precision: Precision) { @@ -147,60 +131,24 @@ impl VTable for FSST { } } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(FSSTMetadata { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some(FSSTMetadata { uncompressed_lengths_ptype: array.uncompressed_lengths().dtype().as_ptype().into(), codes_offsets_ptype: array.codes.offsets().dtype().as_ptype().into(), - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + } + .encode_to_vec())) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata( - as DeserializeMetadata>::deserialize(bytes)?, - )) - } - - fn append_to_builder( - array: ArrayView<'_, Self>, - builder: &mut dyn ArrayBuilder, - ctx: &mut ExecutionCtx, - ) -> VortexResult<()> { - let Some(builder) = builder.as_any_mut().downcast_mut::() else { - builder.extend_from_array( - &array - .array() - .clone() - .execute::(ctx)? - .into_array(), - ); - return Ok(()); - }; - - // Decompress the whole block of data into a new buffer, and create some views - // from it instead. - let (buffers, views) = fsst_decode_views(&array, builder.completed_block_count(), ctx)?; - - builder.push_buffer_and_adjusted_views(&buffers, &views, array.array().validity_mask()?); - Ok(()) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = FSSTMetadata::decode(metadata)?; let symbols = Buffer::::from_byte_buffer(buffers[0].clone().try_to_host_sync()?); let symbol_lengths = Buffer::::from_byte_buffer(buffers[1].clone().try_to_host_sync()?); @@ -222,18 +170,18 @@ impl VTable for FSST { let uncompressed_lengths = children.get( 1, &DType::Primitive( - metadata.0.get_uncompressed_lengths_ptype()?, + metadata.get_uncompressed_lengths_ptype()?, Nullability::NonNullable, ), len, )?; return FSSTData::try_new( - dtype.clone(), symbols, symbol_lengths, codes, uncompressed_lengths, + dtype, ); } @@ -242,7 +190,7 @@ impl VTable for FSST { let uncompressed_lengths = children.get( 0, &DType::Primitive( - metadata.0.get_uncompressed_lengths_ptype()?, + metadata.get_uncompressed_lengths_ptype()?, Nullability::NonNullable, ), len, @@ -276,11 +224,11 @@ impl VTable for FSST { )?; return FSSTData::try_new( - dtype.clone(), symbols, symbol_lengths, codes, uncompressed_lengths, + dtype, ); } @@ -312,7 +260,7 @@ impl VTable for FSST { .vortex_expect("FSSTArray requires codes_offsets slot"); let codes_validity = match &slots[CODES_VALIDITY_SLOT] { Some(v) => Validity::Array(v.clone()), - None => Validity::from(array.dtype.nullability()), + None => Validity::from(array.codes.dtype().nullability()), }; let codes = VarBinArray::try_new( codes_offsets, @@ -328,7 +276,31 @@ impl VTable for FSST { } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { - canonicalize_fsst(&array, ctx).map(ExecutionResult::done) + canonicalize_fsst(array.as_view(), ctx).map(ExecutionResult::done) + } + + fn append_to_builder( + array: ArrayView<'_, Self>, + builder: &mut dyn ArrayBuilder, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let Some(builder) = builder.as_any_mut().downcast_mut::() else { + builder.extend_from_array( + &array + .array() + .clone() + .execute::(ctx)? + .into_array(), + ); + return Ok(()); + }; + + // Decompress the whole block of data into a new buffer, and create some views + // from it instead. + let (buffers, views) = fsst_decode_views(&array, builder.completed_block_count(), ctx)?; + + builder.push_buffer_and_adjusted_views(&buffers, &views, array.array().validity_mask()?); + Ok(()) } fn execute_parent( @@ -361,7 +333,6 @@ pub(crate) const SLOT_NAMES: [&str; NUM_SLOTS] = #[derive(Clone)] pub struct FSSTData { - dtype: DType, symbols: Buffer, symbol_lengths: Buffer, codes: VarBinArray, @@ -369,7 +340,6 @@ pub struct FSSTData { codes_array: ArrayRef, /// Lengths of the original values before compression, can be compressed. slots: Vec>, - stats_set: ArrayStats, /// Memoized compressor used for push-down of compute by compressing the RHS. compressor: Arc Compressor + Send>>>, @@ -378,7 +348,6 @@ pub struct FSSTData { impl Debug for FSSTData { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("FSSTArray") - .field("dtype", &self.dtype) .field("symbols", &self.symbols) .field("symbol_lengths", &self.symbol_lengths) .field("codes", &self.codes) @@ -401,13 +370,28 @@ impl FSST { codes: VarBinArray, uncompressed_lengths: ArrayRef, ) -> VortexResult { - Array::try_from_data(FSSTData::try_new( - dtype, + let len = codes.len(); + let data = FSSTData::try_new( symbols, symbol_lengths, codes, uncompressed_lengths, - )?) + &dtype, + )?; + Array::try_from_parts(ArrayNew::new(FSST, dtype, len, data)) + } + + pub(crate) unsafe fn new_unchecked( + dtype: DType, + symbols: Buffer, + symbol_lengths: Buffer, + codes: VarBinArray, + uncompressed_lengths: ArrayRef, + ) -> FSSTArray { + let len = codes.len(); + let data = unsafe { FSSTData::new_unchecked(symbols, symbol_lengths, codes, uncompressed_lengths) }; + Array::try_from_parts(ArrayNew::new(FSST, dtype, len, data)) + .vortex_expect("pre-validated FSST parts must be valid") } } @@ -421,12 +405,42 @@ impl FSSTData { /// Each code corresponds either to a symbol, or to the "escape code", /// which tells the decoder to emit the following byte without doing a table lookup. pub fn try_new( - dtype: DType, symbols: Buffer, symbol_lengths: Buffer, codes: VarBinArray, uncompressed_lengths: ArrayRef, + dtype: &DType, ) -> VortexResult { + Self::validate_parts(&symbols, &symbol_lengths, &codes, &uncompressed_lengths, dtype, codes.len())?; + + // SAFETY: all components validated above + unsafe { Ok(Self::new_unchecked(symbols, symbol_lengths, codes, uncompressed_lengths)) } + } + + pub fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { + Self::validate_parts( + &self.symbols, + &self.symbol_lengths, + &self.codes, + self.uncompressed_lengths(), + dtype, + len, + ) + } + + fn validate_parts( + symbols: &Buffer, + symbol_lengths: &Buffer, + codes: &VarBinArray, + uncompressed_lengths: &ArrayRef, + dtype: &DType, + len: usize, + ) -> VortexResult<()> { + vortex_ensure!( + matches!(dtype, DType::Binary(_) | DType::Utf8(_)), + "FSST arrays must be Binary or Utf8, found {dtype}" + ); + // Check: symbols must not have length > MAX_CODE if symbols.len() > 255 { vortex_bail!(InvalidArgument: "symbols array must have length <= 255"); @@ -435,7 +449,11 @@ impl FSSTData { vortex_bail!(InvalidArgument: "symbols and symbol_lengths arrays must have same length"); } - if uncompressed_lengths.len() != codes.len() { + if codes.len() != len { + vortex_bail!(InvalidArgument: "codes must have same len as outer array"); + } + + if uncompressed_lengths.len() != len { vortex_bail!(InvalidArgument: "uncompressed_lengths must be same len as codes"); } @@ -447,21 +465,14 @@ impl FSSTData { if !matches!(codes.dtype(), DType::Binary(_)) { vortex_bail!(InvalidArgument: "codes array must be DType::Binary type"); } - - // SAFETY: all components validated above - unsafe { - Ok(Self::new_unchecked( - dtype, - symbols, - symbol_lengths, - codes, - uncompressed_lengths, - )) + if codes.dtype().nullability() != dtype.nullability() { + vortex_bail!(InvalidArgument: "codes nullability must match outer dtype nullability"); } + + Ok(()) } pub(crate) unsafe fn new_unchecked( - dtype: DType, symbols: Buffer, symbol_lengths: Buffer, codes: VarBinArray, @@ -478,7 +489,6 @@ impl FSSTData { let codes_validity_slot = validity_to_child(&codes.validity(), codes.len()); Self { - dtype, symbols, symbol_lengths, codes, @@ -488,7 +498,6 @@ impl FSSTData { codes_offsets_slot, codes_validity_slot, ], - stats_set: Default::default(), compressor, } } @@ -503,11 +512,6 @@ impl FSSTData { self.codes.len() == 0 } - /// Returns the logical data type of the array. - pub fn dtype(&self) -> &DType { - &self.dtype - } - /// Access the symbol table array pub fn symbols(&self) -> &Buffer { &self.symbols diff --git a/encodings/fsst/src/canonical.rs b/encodings/fsst/src/canonical.rs index 41df50f913b..64fb8ea2117 100644 --- a/encodings/fsst/src/canonical.rs +++ b/encodings/fsst/src/canonical.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use vortex_array::ArrayRef; +use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::PrimitiveArray; @@ -17,13 +18,14 @@ use vortex_buffer::ByteBuffer; use vortex_buffer::ByteBufferMut; use vortex_error::VortexResult; +use crate::FSST; use crate::FSSTData; pub(super) fn canonicalize_fsst( - array: &FSSTData, + array: ArrayView<'_, FSST>, ctx: &mut ExecutionCtx, ) -> VortexResult { - let (buffers, views) = fsst_decode_views(array, 0, ctx)?; + let (buffers, views) = fsst_decode_views(array.data(), 0, ctx)?; // SAFETY: FSST already validates the bytes for binary/UTF-8. We build views directly on // top of them, so the view pointers will all be valid. Ok(unsafe { diff --git a/encodings/fsst/src/compress.rs b/encodings/fsst/src/compress.rs index eb016789584..53772e82636 100644 --- a/encodings/fsst/src/compress.rs +++ b/encodings/fsst/src/compress.rs @@ -14,8 +14,8 @@ use vortex_buffer::BufferMut; use vortex_error::VortexExpect; /// Compress a string array using FSST. +use crate::FSST; use crate::FSSTArray; -use crate::FSSTData; pub fn fsst_compress>( strings: A, len: usize, @@ -102,11 +102,8 @@ where let uncompressed_lengths = uncompressed_lengths.into_array(); - FSSTArray::try_from_data( - FSSTData::try_new(dtype, symbols, symbol_lengths, codes, uncompressed_lengths) - .vortex_expect("building FSSTArray from parts"), - ) - .vortex_expect("FSSTData is always valid") + FSST::try_new(dtype, symbols, symbol_lengths, codes, uncompressed_lengths) + .vortex_expect("FSST parts must be valid") } #[cfg(test)] diff --git a/encodings/fsst/src/compute/cast.rs b/encodings/fsst/src/compute/cast.rs index 496d5bf3f92..82034daab21 100644 --- a/encodings/fsst/src/compute/cast.rs +++ b/encodings/fsst/src/compute/cast.rs @@ -11,7 +11,6 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::FSST; -use crate::FSSTData; impl CastReduce for FSST { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // FSST is a string compression encoding. @@ -25,7 +24,7 @@ impl CastReduce for FSST { .cast(array.codes().dtype().with_nullability(dtype.nullability()))?; Ok(Some( - FSSTData::try_new( + FSST::try_new( dtype.clone(), array.symbols().clone(), array.symbol_lengths().clone(), diff --git a/encodings/fsst/src/compute/compare.rs b/encodings/fsst/src/compute/compare.rs index cc0c563a887..14ea2cbe4d6 100644 --- a/encodings/fsst/src/compute/compare.rs +++ b/encodings/fsst/src/compute/compare.rs @@ -21,8 +21,6 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use crate::FSST; -use crate::FSSTData; - impl CompareKernel for FSST { fn compare( lhs: ArrayView<'_, Self>, @@ -31,7 +29,7 @@ impl CompareKernel for FSST { ctx: &mut ExecutionCtx, ) -> VortexResult> { match rhs.as_constant() { - Some(constant) => compare_fsst_constant(&lhs, &constant, operator, ctx), + Some(constant) => compare_fsst_constant(lhs, &constant, operator, ctx), // Otherwise, fall back to the default comparison behavior. _ => Ok(None), } @@ -40,7 +38,7 @@ impl CompareKernel for FSST { /// Specialized compare function implementation used when performing against a constant fn compare_fsst_constant( - left: &FSSTData, + left: ArrayView<'_, FSST>, right: &Scalar, operator: CompareOperator, ctx: &mut ExecutionCtx, @@ -78,7 +76,7 @@ fn compare_fsst_constant( return Ok(Some( BoolArray::new( buffer, - Validity::copy_from_array(&left.clone().into_array())? + Validity::copy_from_array(left.array())? .union_nullability(right.dtype().nullability()), ) .into_array(), diff --git a/encodings/fsst/src/compute/filter.rs b/encodings/fsst/src/compute/filter.rs index de58a83d5c4..0a2aa130d7a 100644 --- a/encodings/fsst/src/compute/filter.rs +++ b/encodings/fsst/src/compute/filter.rs @@ -12,7 +12,6 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::FSST; -use crate::FSSTData; impl FilterKernel for FSST { fn filter( @@ -31,7 +30,7 @@ impl FilterKernel for FSST { .vortex_expect("must be VarBin"); Ok(Some( - FSSTData::try_new( + FSST::try_new( array.dtype().clone(), array.symbols().clone(), array.symbol_lengths().clone(), diff --git a/encodings/fsst/src/compute/mod.rs b/encodings/fsst/src/compute/mod.rs index 7ca91c4c331..1bc1df71a29 100644 --- a/encodings/fsst/src/compute/mod.rs +++ b/encodings/fsst/src/compute/mod.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use crate::FSSTData; mod cast; mod compare; mod filter; @@ -28,7 +27,7 @@ impl TakeExecute for FSST { _ctx: &mut ExecutionCtx, ) -> VortexResult> { Ok(Some( - FSSTData::try_new( + FSST::try_new( array .dtype() .clone() diff --git a/encodings/fsst/src/slice.rs b/encodings/fsst/src/slice.rs index b30e574e9bd..5e0dd21a9f3 100644 --- a/encodings/fsst/src/slice.rs +++ b/encodings/fsst/src/slice.rs @@ -12,14 +12,13 @@ use vortex_error::VortexResult; use vortex_error::vortex_err; use crate::FSST; -use crate::FSSTData; impl SliceReduce for FSST { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { // SAFETY: slicing the `codes` leaves the symbol table intact Ok(Some( unsafe { - FSSTData::new_unchecked( + FSST::new_unchecked( array.dtype().clone(), array.symbols().clone(), array.symbol_lengths().clone(), diff --git a/encodings/parquet-variant/src/array.rs b/encodings/parquet-variant/src/array.rs index f07ae0e847f..a0c2a47d286 100644 --- a/encodings/parquet-variant/src/array.rs +++ b/encodings/parquet-variant/src/array.rs @@ -7,6 +7,8 @@ use arrow_array::Array as ArrowArray; use arrow_array::ArrayRef as ArrowArrayRef; use arrow_schema::Field; use parquet_variant_compute::VariantArray as ArrowVariantArray; +use vortex_array::Array; +use vortex_array::ArrayNew; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; @@ -15,7 +17,6 @@ use vortex_array::arrow::ArrowArrayExecutor; use vortex_array::arrow::FromArrowArray; use vortex_array::arrow::to_arrow_null_buffer; use vortex_array::dtype::DType; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable::validity_to_child; use vortex_buffer::BitBuffer; @@ -24,6 +25,8 @@ use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_ensure_eq; +use crate::ParquetVariant; + /// The validity bitmap indicating which elements are non-null. pub(crate) const VALIDITY_SLOT: usize = 0; /// The metadata array for the Parquet variant values. @@ -63,10 +66,22 @@ pub(crate) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity", "metadata", "value /// (which includes nullability). #[derive(Clone, Debug)] pub struct ParquetVariantData { - pub(crate) dtype: DType, pub(crate) validity: Validity, pub(crate) slots: Vec>, - pub(crate) stats_set: ArrayStats, +} + +impl ParquetVariant { + pub fn try_new( + validity: Validity, + metadata: ArrayRef, + value: Option, + typed_value: Option, + ) -> VortexResult> { + let len = metadata.len(); + let dtype = DType::Variant(validity.nullability()); + let data = ParquetVariantData::try_new(validity, metadata, value, typed_value)?; + Array::try_from_parts(ArrayNew::new(ParquetVariant, dtype, len, data)) + } } impl ParquetVariantData { @@ -77,39 +92,86 @@ impl ParquetVariantData { value: Option, typed_value: Option, ) -> VortexResult { + let len = metadata.len(); + let dtype = DType::Variant(validity.nullability()); + Self::validate_parts( + &validity, + &metadata, + value.as_ref(), + typed_value.as_ref(), + &dtype, + len, + )?; + + let validity_child = validity_to_child(&validity, len); + let slots = vec![validity_child, Some(metadata), value, typed_value]; + + Ok(Self { validity, slots }) + } + + pub(crate) fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { + Self::validate_parts( + &self.validity, + self.metadata_array(), + self.value_array(), + self.typed_value_array(), + dtype, + len, + ) + } + + fn validate_parts( + validity: &Validity, + metadata: &ArrayRef, + value: Option<&ArrayRef>, + typed_value: Option<&ArrayRef>, + dtype: &DType, + len: usize, + ) -> VortexResult<()> { + vortex_ensure!( + matches!(dtype, DType::Variant(_)), + "Expected Variant DType, found {dtype}" + ); vortex_ensure!( value.is_some() || typed_value.is_some(), "at least one of value or typed_value must be present" ); - let len = metadata.len(); + + vortex_ensure_eq!( + dtype.nullability(), + validity.nullability(), + "variant dtype nullability must match validity nullability" + ); + vortex_ensure_eq!( + metadata.dtype(), + &DType::Binary(vortex_array::dtype::Nullability::NonNullable), + "metadata dtype must be non-nullable binary" + ); + vortex_ensure_eq!(metadata.len(), len, "metadata length must match array length"); + if let Some(validity_len) = validity.maybe_len() { vortex_ensure_eq!( validity_len, len, - "validity length must match metadata length" + "validity length must match array length" ); } - if let Some(ref v) = value { - vortex_ensure_eq!(v.len(), len, "value length must match metadata length"); + if let Some(v) = value { + vortex_ensure!( + matches!(v.dtype(), DType::Binary(_)), + "value dtype must be binary, found {}", + v.dtype() + ); + vortex_ensure_eq!(v.len(), len, "value length must match array length"); } - if let Some(ref tv) = typed_value { + if let Some(tv) = typed_value { vortex_ensure_eq!( tv.len(), len, - "typed_value length must match metadata length" + "typed_value length must match array length" ); } - let nullability = validity.nullability(); - - let validity_child = validity_to_child(&validity, len); - let slots = vec![validity_child, Some(metadata), value, typed_value]; - - Ok(Self { - dtype: DType::Variant(nullability), - validity, - slots, - stats_set: ArrayStats::default(), - }) + Ok(()) } /// Returns a reference to the metadata child array. @@ -168,7 +230,7 @@ impl ParquetVariantData { .map(|tv| ArrayRef::from_arrow(tv.as_ref(), typed_value_nullable)) .transpose()?; - let pv = ParquetVariantData::try_new(validity, metadata, value, typed_value)?; + let pv = ParquetVariant::try_new(validity, metadata, value, typed_value)?; Ok(VariantArray::new(pv.into_array()).into_array()) } diff --git a/encodings/parquet-variant/src/kernel.rs b/encodings/parquet-variant/src/kernel.rs index d921f97a325..69e922e1548 100644 --- a/encodings/parquet-variant/src/kernel.rs +++ b/encodings/parquet-variant/src/kernel.rs @@ -18,7 +18,6 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::ParquetVariant; -use crate::array::ParquetVariantData; pub(crate) static PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ ParentKernelSet::lift(&FilterExecuteAdaptor(ParquetVariant)), @@ -43,7 +42,7 @@ impl SliceKernel for ParquetVariant { .map(|tv| tv.slice(range)) .transpose()?; Ok(Some( - ParquetVariantData::try_new(validity, metadata, value, typed_value)?.into_array(), + ParquetVariant::try_new(validity, metadata, value, typed_value)?.into_array(), )) } } @@ -65,7 +64,7 @@ impl FilterKernel for ParquetVariant { .map(|tv| tv.filter(mask.clone())) .transpose()?; Ok(Some( - ParquetVariantData::try_new(validity, metadata, value, typed_value)?.into_array(), + ParquetVariant::try_new(validity, metadata, value, typed_value)?.into_array(), )) } } @@ -87,7 +86,7 @@ impl TakeExecute for ParquetVariant { .map(|tv| tv.take(indices.clone())) .transpose()?; Ok(Some( - ParquetVariantData::try_new(validity, metadata, value, typed_value)?.into_array(), + ParquetVariant::try_new(validity, metadata, value, typed_value)?.into_array(), )) } } diff --git a/encodings/parquet-variant/src/lib.rs b/encodings/parquet-variant/src/lib.rs index 3f1f7cb68a6..42c97d28870 100644 --- a/encodings/parquet-variant/src/lib.rs +++ b/encodings/parquet-variant/src/lib.rs @@ -33,4 +33,3 @@ mod vtable; pub use array::ParquetVariantData; pub use vtable::ParquetVariant; pub use vtable::ParquetVariantArray; -pub use vtable::ParquetVariantMetadata; diff --git a/encodings/parquet-variant/src/operations.rs b/encodings/parquet-variant/src/operations.rs index 6ccbfeb6d75..4b2d26b77a4 100644 --- a/encodings/parquet-variant/src/operations.rs +++ b/encodings/parquet-variant/src/operations.rs @@ -61,7 +61,7 @@ impl OperationsVTable for ParquetVariant { )?; Scalar::try_new( - array.dtype.clone(), + array.dtype().clone(), Some(ScalarValue::Variant(Box::new(inner))), ) } diff --git a/encodings/parquet-variant/src/vtable.rs b/encodings/parquet-variant/src/vtable.rs index 8815734e0c9..e462afe9096 100644 --- a/encodings/parquet-variant/src/vtable.rs +++ b/encodings/parquet-variant/src/vtable.rs @@ -20,7 +20,6 @@ use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; @@ -46,23 +45,6 @@ impl ParquetVariant { pub const ID: ArrayId = ArrayId::new_ref("vortex.parquet.variant"); } -/// Serialized metadata for a [`ParquetVariantArray`]. -#[derive(Clone, Debug)] -pub struct ParquetVariantMetadata { - /// Whether the un-shredded `value` child is present. - pub has_value: bool, - /// Whether the `value` child is nullable. - /// - /// In partially-shredded layouts, rows whose data lives entirely in `typed_value` have a - /// null `value` slot, so the Arrow field is marked nullable. This flag preserves that - /// distinction across serialization round-trips. - pub value_nullable: bool, - /// DType of the shredded `typed_value`, if present. - /// - /// This is required to deserialize non-variant shredded children. - pub typed_value_dtype: Option, -} - #[derive(Clone, prost::Message)] struct ParquetVariantMetadataProto { /// Whether the un-shredded `value` child is present. @@ -80,28 +62,15 @@ vtable!(ParquetVariant, ParquetVariant, ParquetVariantData); impl VTable for ParquetVariant { type ArrayData = ParquetVariantData; - type Metadata = ParquetVariantMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromValidityHelper; - fn vtable(_array: &Self::ArrayData) -> &Self { - &ParquetVariant - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ParquetVariantData) -> usize { - array.metadata_array().len() - } - - fn dtype(array: &ParquetVariantData) -> &DType { - &array.dtype - } - - fn stats(array: &ParquetVariantData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate(dtype, len) } fn array_hash(array: &ParquetVariantData, state: &mut H, precision: Precision) { @@ -167,64 +136,50 @@ impl VTable for ParquetVariant { SLOT_NAMES[idx].to_string() } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ParquetVariantMetadata { - has_value: array.value_array().is_some(), - value_nullable: array.value_array().is_some_and(|v| v.dtype().is_nullable()), - typed_value_dtype: array.typed_value_array().map(|tv| tv.dtype().clone()), - }) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - let typed_value_dtype = metadata - .typed_value_dtype - .as_ref() - .map(|dtype| dtype.try_into()) + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + let typed_value_dtype = array + .typed_value_array() + .map(|tv| tv.dtype().try_into()) .transpose()?; Ok(Some( ParquetVariantMetadataProto { - has_value: metadata.has_value, + has_value: array.value_array().is_some(), typed_value_dtype, - value_nullable: metadata.value_nullable, + value_nullable: array.value_array().is_some_and(|v| v.dtype().is_nullable()), } .encode_to_vec(), )) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], + &self, + dtype: &DType, + len: usize, + metadata: &[u8], + buffers: &[BufferHandle], + children: &dyn ArrayChildren, session: &VortexSession, - ) -> VortexResult { - let proto = ParquetVariantMetadataProto::decode(bytes)?; + ) -> VortexResult { + vortex_ensure!( + buffers.is_empty(), + "ParquetVariantArray expects 0 buffers, got {}", + buffers.len() + ); + + let proto = ParquetVariantMetadataProto::decode(metadata)?; let typed_value_dtype = match proto.typed_value_dtype.as_ref() { Some(dtype) => Some(DType::from_proto(dtype, session)?), None => None, }; - Ok(ParquetVariantMetadata { - has_value: proto.has_value, - value_nullable: proto.value_nullable, - typed_value_dtype, - }) - } - fn build( - dtype: &DType, - len: usize, - metadata: &Self::Metadata, - _buffers: &[BufferHandle], - children: &dyn ArrayChildren, - ) -> VortexResult { vortex_ensure!(matches!(dtype, DType::Variant(_)), "Expected Variant DType"); - let has_typed_value = metadata.typed_value_dtype.is_some(); + let has_typed_value = typed_value_dtype.is_some(); vortex_ensure!( - metadata.has_value || has_typed_value, + proto.has_value || has_typed_value, "At least one of value or typed_value must be present" ); - let expected_children = 1 + metadata.has_value as usize + has_typed_value as usize; + let expected_children = 1 + proto.has_value as usize + has_typed_value as usize; vortex_ensure!( children.len() == expected_children || children.len() == expected_children + 1, "Expected {} or {} children, got {}", @@ -242,10 +197,10 @@ impl VTable for ParquetVariant { children.get(child_idx, &DType::Binary(Nullability::NonNullable), len)?; child_idx += 1; - let value = if metadata.has_value { + let value = if proto.has_value { let v = children.get( child_idx, - &DType::Binary(metadata.value_nullable.into()), + &DType::Binary(proto.value_nullable.into()), len, )?; child_idx += 1; @@ -256,8 +211,7 @@ impl VTable for ParquetVariant { let typed_value = if has_typed_value { // typed_value can be any type — primitive, list, struct, etc. - let dtype = metadata - .typed_value_dtype + let dtype = typed_value_dtype .clone() .ok_or_else(|| vortex_err!("typed_value_dtype missing for typed_value child"))?; let tv = children.get(child_idx, &dtype, len)?; @@ -357,7 +311,7 @@ mod tests { let inner_metadata = VarBinViewArray::from_iter_bin([b"\x01\x00", b"\x01\x00", b"\x01\x00"]).into_array(); let inner_value = VarBinViewArray::from_iter_bin([b"\x02", b"\x03", b"\x04"]).into_array(); - let inner_pv = ParquetVariantData::try_new( + let inner_pv = ParquetVariant::try_new( Validity::NonNullable, inner_metadata, Some(inner_value), @@ -366,7 +320,7 @@ mod tests { .unwrap(); let typed_value = VariantArray::new(inner_pv.into_array()).into_array(); - let outer_pv = ParquetVariantData::try_new( + let outer_pv = ParquetVariant::try_new( Validity::NonNullable, outer_metadata, None, @@ -389,7 +343,7 @@ mod tests { let value = VarBinViewArray::from_iter_bin([b"\x10", b"\x11", b"\x12"]).into_array(); let validity = Validity::from(BitBuffer::from_iter([true, false, true])); - let pv = ParquetVariantData::try_new(validity, metadata, Some(value), None).unwrap(); + let pv = ParquetVariant::try_new(validity, metadata, Some(value), None).unwrap(); let array = pv.into_array(); let decoded = roundtrip(array.clone()); @@ -406,7 +360,7 @@ mod tests { VarBinViewArray::from_iter_bin([b"\x01\x00", b"\x01\x00", b"\x01\x00"]).into_array(); let typed_value = buffer![10i32, 20, 30].into_array(); - let outer_pv = ParquetVariantData::try_new( + let outer_pv = ParquetVariant::try_new( Validity::NonNullable, outer_metadata, None, diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index d8c6875fa39..ea1d19e49b5 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -8,6 +8,7 @@ use std::sync::Arc; use itertools::Itertools as _; use prost::Message as _; use vortex_array::Array; +use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; @@ -19,7 +20,6 @@ use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; use vortex_array::Precision; -use vortex_array::ProstMetadata; use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; @@ -32,7 +32,6 @@ use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; @@ -85,28 +84,15 @@ vtable!(Zstd, Zstd, ZstdData); impl VTable for Zstd { type ArrayData = ZstdData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromValiditySliceHelper; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Zstd - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ZstdData) -> usize { - array.slice_stop - array.slice_start - } - - fn dtype(array: &ZstdData) -> &DType { - &array.dtype - } - - fn stats(array: &ZstdData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate(dtype, len) } fn array_hash(array: &ZstdData, state: &mut H, precision: Precision) { @@ -179,31 +165,20 @@ impl VTable for Zstd { } } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(array.metadata.clone())) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.0.encode_to_vec())) + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some(array.metadata.clone().encode_to_vec())) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata(ZstdMetadata::decode(bytes)?)) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = ZstdMetadata::decode(metadata)?; let validity = if children.is_empty() { Validity::from(dtype.nullability()) } else if children.len() == 1 { @@ -213,7 +188,7 @@ impl VTable for Zstd { vortex_bail!("ZstdArray expected 0 or 1 child, got {}", children.len()); }; - let (dictionary_buffer, compressed_buffers) = if metadata.0.dictionary_size == 0 { + let (dictionary_buffer, compressed_buffers) = if metadata.dictionary_size == 0 { // no dictionary ( None, @@ -236,8 +211,7 @@ impl VTable for Zstd { Ok(ZstdData::new( dictionary_buffer, compressed_buffers, - dtype.clone(), - metadata.0.clone(), + metadata, len, validity, )) @@ -261,7 +235,7 @@ impl VTable for Zstd { array.unsliced_validity = match &slots[VALIDITY_SLOT] { Some(arr) => Validity::Array(arr.clone()), - None => Validity::from(array.dtype.nullability()), + None => Validity::from(array.unsliced_validity.nullability()), }; array.slots = slots; @@ -269,8 +243,7 @@ impl VTable for Zstd { } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { - array - .decompress(ctx)? + Zstd::decompress(&array, ctx)? .execute::(ctx) .map(ExecutionResult::done) } @@ -290,17 +263,21 @@ pub struct Zstd; impl Zstd { pub const ID: ArrayId = ArrayId::new_ref("vortex.zstd"); + pub fn try_new(dtype: DType, data: ZstdData) -> VortexResult { + let len = data.len(); + Array::try_from_parts(ArrayNew::new(Zstd, dtype, len, data)) + } + /// Compress a [`VarBinViewArray`] using Zstd without a dictionary. pub fn from_var_bin_view_without_dict( vbv: &VarBinViewArray, level: i32, values_per_frame: usize, ) -> VortexResult { - Array::try_from_data(ZstdData::from_var_bin_view_without_dict( - vbv, - level, - values_per_frame, - )?) + Self::try_new( + vbv.dtype().clone(), + ZstdData::from_var_bin_view_without_dict(vbv, level, values_per_frame)?, + ) } /// Compress a [`PrimitiveArray`] using Zstd. @@ -309,7 +286,10 @@ impl Zstd { level: i32, values_per_frame: usize, ) -> VortexResult { - Array::try_from_data(ZstdData::from_primitive(parray, level, values_per_frame)?) + Self::try_new( + parray.dtype().clone(), + ZstdData::from_primitive(parray, level, values_per_frame)?, + ) } /// Compress a [`VarBinViewArray`] using Zstd. @@ -318,7 +298,30 @@ impl Zstd { level: i32, values_per_frame: usize, ) -> VortexResult { - Array::try_from_data(ZstdData::from_var_bin_view(vbv, level, values_per_frame)?) + Self::try_new( + vbv.dtype().clone(), + ZstdData::from_var_bin_view(vbv, level, values_per_frame)?, + ) + } + + pub fn decompress(array: &ZstdArray, ctx: &mut ExecutionCtx) -> VortexResult { + array.data().decompress(array.dtype(), ctx) + } + + pub fn into_parts(array: ZstdArray) -> ZstdArrayParts { + let dtype = array.dtype().clone(); + let data = array.into_data(); + + ZstdArrayParts { + dictionary: data.dictionary, + frames: data.frames, + metadata: data.metadata, + dtype, + validity: data.unsliced_validity, + n_rows: data.unsliced_n_rows, + slice_start: data.slice_start, + slice_stop: data.slice_stop, + } } } @@ -332,11 +335,9 @@ pub struct ZstdData { pub(crate) dictionary: Option, pub(crate) frames: Vec, pub(crate) metadata: ZstdMetadata, - dtype: DType, pub(crate) unsliced_validity: Validity, unsliced_n_rows: usize, pub(super) slots: Vec>, - stats_set: ArrayStats, slice_start: usize, slice_stop: usize, } @@ -462,7 +463,6 @@ impl ZstdData { pub fn new( dictionary: Option, frames: Vec, - dtype: DType, metadata: ZstdMetadata, n_rows: usize, validity: Validity, @@ -472,16 +472,91 @@ impl ZstdData { dictionary, frames, metadata, - dtype, unsliced_validity: validity, unsliced_n_rows: n_rows, slots: vec![validity_slot], - stats_set: Default::default(), slice_start: 0, slice_stop: n_rows, } } + pub fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + matches!(dtype, DType::Primitive(..) | DType::Binary(_) | DType::Utf8(_)), + "Unsupported dtype for Zstd array: {dtype}" + ); + vortex_ensure!( + self.slice_start <= self.slice_stop, + "Invalid slice range {}..{}", + self.slice_start, + self.slice_stop + ); + vortex_ensure!( + self.slice_stop <= self.unsliced_n_rows, + "Slice stop {} exceeds unsliced row count {}", + self.slice_stop, + self.unsliced_n_rows + ); + vortex_ensure!( + self.slice_stop - self.slice_start == len, + "Slice length {} does not match array length {}", + self.slice_stop - self.slice_start, + len + ); + if let Some(validity_len) = self.unsliced_validity.maybe_len() { + vortex_ensure!( + validity_len == self.unsliced_n_rows, + "Validity length {} does not match unsliced row count {}", + validity_len, + self.unsliced_n_rows + ); + } + + match &self.dictionary { + Some(dictionary) => vortex_ensure!( + usize::try_from(self.metadata.dictionary_size)? == dictionary.len(), + "Dictionary size metadata {} does not match buffer size {}", + self.metadata.dictionary_size, + dictionary.len() + ), + None => vortex_ensure!( + self.metadata.dictionary_size == 0, + "Dictionary metadata present without dictionary buffer" + ), + } + vortex_ensure!( + self.frames.len() == self.metadata.frames.len(), + "Frame count {} does not match metadata frame count {}", + self.frames.len(), + self.metadata.frames.len() + ); + + Ok(()) + } + + pub(crate) fn with_slice(&self, start: usize, stop: usize) -> Self { + let new_start = self.slice_start + start; + let new_stop = self.slice_start + stop; + + assert!( + new_start <= self.slice_stop, + "new slice start {new_start} exceeds end {}", + self.slice_stop + ); + + assert!( + new_stop <= self.slice_stop, + "new slice stop {new_stop} exceeds end {}", + self.slice_stop + ); + + Self { + slice_start: new_start, + slice_stop: new_stop, + ..self.clone() + } + } + fn compress_values( value_bytes: &ByteBuffer, frame_byte_starts: &[usize], @@ -585,7 +660,6 @@ impl ZstdData { values_per_frame: usize, use_dictionary: bool, ) -> VortexResult { - let dtype = parray.dtype().clone(); let byte_width = parray.ptype().byte_width(); // We compress only the valid elements. @@ -629,7 +703,6 @@ impl ZstdData { Ok(ZstdData::new( dictionary, frames, - dtype, metadata, parray.len(), parray.validity(), @@ -682,8 +755,6 @@ impl ZstdData { // to separate the binary data and lengths as two separate streams, but // this approach is simpler and can be best in cases when there is // mutual information between strings and their lengths. - let dtype = vbv.dtype().clone(); - // We compress only the valid elements. let (value_bytes, value_byte_indices) = collect_valid_vbv(vbv)?; let n_values = value_byte_indices.len(); @@ -720,7 +791,6 @@ impl ZstdData { Ok(ZstdData::new( dictionary, frames, - dtype, metadata, vbv.len(), vbv.validity(), @@ -752,18 +822,18 @@ impl ZstdData { .ok_or_else(|| vortex_err!("Zstd can only encode Primitive and VarBinView arrays")) } - fn byte_width(&self) -> usize { - if self.dtype.is_primitive() { - self.dtype.as_ptype().byte_width() + fn byte_width(dtype: &DType) -> usize { + if dtype.is_primitive() { + dtype.as_ptype().byte_width() } else { 1 } } - pub fn decompress(&self, ctx: &mut ExecutionCtx) -> VortexResult { + fn decompress(&self, dtype: &DType, ctx: &mut ExecutionCtx) -> VortexResult { // To start, we figure out which frames we need to decompress, and with // what row offset into the first such frame. - let byte_width = self.byte_width(); + let byte_width = Self::byte_width(dtype); let slice_n_rows = self.slice_stop - self.slice_start; let slice_value_indices = self .unsliced_validity @@ -845,21 +915,21 @@ impl ZstdData { // // We ensure that the validity of the decompressed array ALWAYS matches the validity // implied by the DType. - if !self.dtype().is_nullable() && !matches!(slice_validity, Validity::NonNullable) { + if !dtype.is_nullable() && !matches!(slice_validity, Validity::NonNullable) { assert!( matches!(slice_validity, Validity::AllValid), "ZSTD array expects to be non-nullable but there are nulls after decompression" ); slice_validity = Validity::NonNullable; - } else if self.dtype.is_nullable() && matches!(slice_validity, Validity::NonNullable) { + } else if dtype.is_nullable() && matches!(slice_validity, Validity::NonNullable) { slice_validity = Validity::AllValid; } // // END OF IMPORTANT BLOCK // - match &self.dtype { + match dtype { DType::Primitive(..) => { let slice_values_buffer = decompressed.slice( (slice_value_idx_start - n_skipped_values) * byte_width @@ -867,7 +937,7 @@ impl ZstdData { ); let primitive = PrimitiveArray::from_values_byte_buffer( slice_values_buffer, - self.dtype.as_ptype(), + dtype.as_ptype(), slice_validity, slice_n_rows, ); @@ -888,14 +958,14 @@ impl ZstdData { VarBinViewArray::new_unchecked( valid_views, Arc::from(buffers), - self.dtype.clone(), + dtype.clone(), slice_validity, ) } .into_array()) } AllOr::None => Ok(ConstantArray::new( - Scalar::null(self.dtype.clone()), + Scalar::null(dtype.clone()), slice_n_rows, ) .into_array()), @@ -916,7 +986,7 @@ impl ZstdData { VarBinViewArray::new_unchecked( views.freeze(), Arc::from(buffers), - self.dtype.clone(), + dtype.clone(), slice_validity, ) } @@ -924,46 +994,7 @@ impl ZstdData { } } } - _ => vortex_panic!("Unsupported dtype for Zstd array: {}", self.dtype), - } - } - - pub(crate) fn _slice(&self, start: usize, stop: usize) -> ZstdArray { - let new_start = self.slice_start + start; - let new_stop = self.slice_start + stop; - - assert!( - new_start <= self.slice_stop, - "new slice start {new_start} exceeds end {}", - self.slice_stop - ); - - assert!( - new_stop <= self.slice_stop, - "new slice stop {new_stop} exceeds end {}", - self.slice_stop - ); - - Array::try_from_data(ZstdData { - slice_start: self.slice_start + start, - slice_stop: self.slice_start + stop, - stats_set: Default::default(), - ..self.clone() - }) - .vortex_expect("ZstdData is always valid") - } - - /// Consumes the array and returns its parts. - pub fn into_parts(self) -> ZstdArrayParts { - ZstdArrayParts { - dictionary: self.dictionary, - frames: self.frames, - metadata: self.metadata, - dtype: self.dtype, - validity: self.unsliced_validity, - n_rows: self.unsliced_n_rows, - slice_start: self.slice_start, - slice_stop: self.slice_stop, + _ => vortex_panic!("Unsupported dtype for Zstd array: {}", dtype), } } @@ -979,12 +1010,6 @@ impl ZstdData { self.slice_stop == self.slice_start } - /// Returns the logical data type of the array. - #[inline] - pub fn dtype(&self) -> &DType { - &self.dtype - } - pub(crate) fn slice_start(&self) -> usize { self.slice_start } @@ -1011,10 +1036,8 @@ impl OperationsVTable for Zstd { _ctx: &mut ExecutionCtx, ) -> VortexResult { let mut ctx = LEGACY_SESSION.create_execution_ctx(); - array - ._slice(index, index + 1) - .decompress(&mut ctx)? - .scalar_at(0) + let sliced = Zstd::try_new(array.dtype().clone(), array.data().with_slice(index, index + 1))?; + Zstd::decompress(&sliced, &mut ctx)?.scalar_at(0) } } diff --git a/encodings/zstd/src/compute/cast.rs b/encodings/zstd/src/compute/cast.rs index feadd131e95..4ad636fe88a 100644 --- a/encodings/zstd/src/compute/cast.rs +++ b/encodings/zstd/src/compute/cast.rs @@ -34,14 +34,16 @@ impl CastReduce for Zstd { (Nullability::NonNullable, Nullability::Nullable) => { // nonnull => null, trivial cast by altering the validity Ok(Some( - ZstdData::new( - array.dictionary.clone(), - array.frames.clone(), + Zstd::try_new( dtype.clone(), - array.metadata.clone(), - array.unsliced_n_rows(), - array.unsliced_validity.clone(), - ) + ZstdData::new( + array.dictionary.clone(), + array.frames.clone(), + array.metadata.clone(), + array.unsliced_n_rows(), + array.unsliced_validity.clone(), + ), + )? .into_array() .slice(array.slice_start()..array.slice_stop())?, )) @@ -60,14 +62,16 @@ impl CastReduce for Zstd { // If there are no nulls, the cast is trivial Ok(Some( - ZstdData::new( - array.dictionary.clone(), - array.frames.clone(), + Zstd::try_new( dtype.clone(), - array.metadata.clone(), - array.unsliced_n_rows(), - array.unsliced_validity.clone(), - ) + ZstdData::new( + array.dictionary.clone(), + array.frames.clone(), + array.metadata.clone(), + array.unsliced_n_rows(), + array.unsliced_validity.clone(), + ), + )? .into_array() .slice(array.slice_start()..array.slice_stop())?, )) diff --git a/encodings/zstd/src/slice.rs b/encodings/zstd/src/slice.rs index fad72ae3279..c5fd5d528ee 100644 --- a/encodings/zstd/src/slice.rs +++ b/encodings/zstd/src/slice.rs @@ -13,6 +13,9 @@ use crate::Zstd; impl SliceReduce for Zstd { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - Ok(Some(array._slice(range.start, range.end).into_array())) + Ok(Some( + Zstd::try_new(array.dtype().clone(), array.data().with_slice(range.start, range.end))? + .into_array(), + )) } } diff --git a/encodings/zstd/src/test.rs b/encodings/zstd/src/test.rs index e95ce025667..f4f78b72ff7 100644 --- a/encodings/zstd/src/test.rs +++ b/encodings/zstd/src/test.rs @@ -32,7 +32,7 @@ fn test_zstd_compress_decompress() { // check full decompression works let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let decompressed = compressed.decompress(&mut ctx).unwrap(); + let decompressed = Zstd::decompress(&compressed, &mut ctx).unwrap(); assert_arrays_eq!(decompressed, PrimitiveArray::from_iter(data)); // check slicing works @@ -78,7 +78,7 @@ fn test_zstd_with_validity_and_multi_frame() { assert_nth_scalar!(compressed, 177, 177); let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let decompressed = compressed.decompress(&mut ctx).unwrap().to_primitive(); + let decompressed = Zstd::decompress(&compressed, &mut ctx).unwrap().to_primitive(); let decompressed_values = decompressed.as_slice::(); assert_eq!(decompressed_values[3], 3); assert_eq!(decompressed_values[177], 177); @@ -121,7 +121,7 @@ fn test_zstd_with_dict() { assert_nth_scalar!(compressed, 199, 199); let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let decompressed = compressed.decompress(&mut ctx).unwrap().to_primitive(); + let decompressed = Zstd::decompress(&compressed, &mut ctx).unwrap().to_primitive(); assert_arrays_eq!(decompressed, PrimitiveArray::from_iter(data)); // check slicing works @@ -193,7 +193,7 @@ fn test_zstd_decompress_var_bin_view() { assert_nth_scalar!(compressed, 4, "baz"); let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let decompressed = compressed.decompress(&mut ctx).unwrap().to_varbinview(); + let decompressed = Zstd::decompress(&compressed, &mut ctx).unwrap().to_varbinview(); assert_nth_scalar!(decompressed, 0, "foo"); assert_nth_scalar!(decompressed, 1, "bar"); assert_nth_scalar!(decompressed, 2, None::); diff --git a/encodings/zstd/src/zstd_buffers.rs b/encodings/zstd/src/zstd_buffers.rs index 4f5b8a9b918..0b1439e27d9 100644 --- a/encodings/zstd/src/zstd_buffers.rs +++ b/encodings/zstd/src/zstd_buffers.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use prost::Message as _; use vortex_array::Array; +use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; @@ -14,15 +15,12 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; -use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; use vortex_array::session::ArraySessionExt; -use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; use vortex_array::vtable::VTable; @@ -44,6 +42,61 @@ pub struct ZstdBuffers; impl ZstdBuffers { pub const ID: ArrayId = ArrayId::new_ref("vortex.zstd_buffers"); + + pub fn try_new(dtype: DType, len: usize, data: ZstdBuffersData) -> VortexResult { + Array::try_from_parts(ArrayNew::new(ZstdBuffers, dtype, len, data)) + } + + pub fn compress(array: &ArrayRef, level: i32) -> VortexResult { + let encoding_id = array.encoding_id(); + let metadata = array + .metadata()? + .ok_or_else(|| vortex_err!("Array does not support serialization"))?; + let buffer_handles = array.buffer_handles(); + let children = array.children(); + + let mut compressed_buffers = Vec::with_capacity(buffer_handles.len()); + let mut uncompressed_sizes = Vec::with_capacity(buffer_handles.len()); + let mut buffer_alignments = Vec::with_capacity(buffer_handles.len()); + + let mut compressor = zstd::bulk::Compressor::new(level)?; + // Compression is currently CPU-only, so we gather all buffers on the host. + for handle in &buffer_handles { + buffer_alignments.push(u32::from(handle.alignment())); + let host_buf = handle.clone().try_to_host_sync()?; + uncompressed_sizes.push(host_buf.len() as u64); + let compressed = compressor.compress(&host_buf)?; + compressed_buffers.push(BufferHandle::new_host(ByteBuffer::from(compressed))); + } + + let data = ZstdBuffersData { + inner_encoding_id: encoding_id, + inner_metadata: metadata, + compressed_buffers, + uncompressed_sizes, + buffer_alignments, + slots: children.into_iter().map(Some).collect(), + }; + let compressed = Self::try_new(array.dtype().clone(), array.len(), data)?; + compressed.statistics().inherit_from(array.statistics()); + Ok(compressed) + } + + pub fn build_inner( + array: &ZstdBuffersArray, + buffer_handles: &[BufferHandle], + session: &VortexSession, + ) -> VortexResult { + array.data().build_inner(array.dtype(), array.len(), buffer_handles, session) + } + + fn decompress_and_build_inner( + array: &ZstdBuffersArray, + session: &VortexSession, + ) -> VortexResult { + let decompressed_buffers = array.data().decompress_buffers()?; + Self::build_inner(array, &decompressed_buffers, session) + } } /// An encoding that ZSTD-compresses the buffers of any wrapped array. @@ -59,9 +112,6 @@ pub struct ZstdBuffersData { uncompressed_sizes: Vec, buffer_alignments: Vec, pub(crate) slots: Vec>, - dtype: DType, - len: usize, - stats_set: ArrayStats, } #[derive(Clone, Debug)] @@ -124,24 +174,6 @@ impl ZstdBuffersDecodePlan { } impl ZstdBuffersData { - /// Returns the length of the array. - #[inline] - pub fn len(&self) -> usize { - self.len - } - - /// Returns whether the array is empty. - #[inline] - pub fn is_empty(&self) -> bool { - self.len == 0 - } - - /// Returns the logical data type of the array. - #[inline] - pub fn dtype(&self) -> &DType { - &self.dtype - } - fn validate(&self) -> VortexResult<()> { vortex_ensure_eq!( self.compressed_buffers.len(), @@ -160,51 +192,6 @@ impl ZstdBuffersData { Ok(()) } - /// Compresses the buffers of the given array using ZSTD. - /// - /// Each buffer of the input array is independently ZSTD-compressed. The children - /// and metadata of the input array are preserved as-is. - pub fn compress(array: &ArrayRef, level: i32) -> VortexResult { - let encoding_id = array.encoding_id(); - let metadata = array - .metadata()? - .ok_or_else(|| vortex_err!("Array does not support serialization"))?; - let buffer_handles = array.buffer_handles(); - let children = array.children(); - - let mut compressed_buffers = Vec::with_capacity(buffer_handles.len()); - let mut uncompressed_sizes = Vec::with_capacity(buffer_handles.len()); - let mut buffer_alignments = Vec::with_capacity(buffer_handles.len()); - - let mut compressor = zstd::bulk::Compressor::new(level)?; - // Compression is currently CPU-only, so we gather all buffers on the host. - for handle in &buffer_handles { - buffer_alignments.push(u32::from(handle.alignment())); - let host_buf = handle.clone().try_to_host_sync()?; - uncompressed_sizes.push(host_buf.len() as u64); - let compressed = compressor.compress(&host_buf)?; - compressed_buffers.push(BufferHandle::new_host(ByteBuffer::from(compressed))); - } - - let compressed = Self { - inner_encoding_id: encoding_id, - inner_metadata: metadata, - compressed_buffers, - uncompressed_sizes, - buffer_alignments, - slots: children.into_iter().map(Some).collect(), - dtype: array.dtype().clone(), - len: array.len(), - stats_set: Default::default(), - }; - let compressed_ref = compressed.clone().into_array(); - compressed - .stats_set - .to_ref(&compressed_ref) - .inherit_from(array.statistics()); - Ok(compressed) - } - fn decompress_buffers(&self) -> VortexResult> { // CPU decode path: zstd::bulk works on host bytes, so compressed buffers are // materialized on the host via `try_to_host_sync`. @@ -252,15 +239,22 @@ impl ZstdBuffersData { Ok(result) } - fn decompress_and_build_inner(&self, session: &VortexSession) -> VortexResult { + fn decompress_and_build_inner( + &self, + dtype: &DType, + len: usize, + session: &VortexSession, + ) -> VortexResult { let decompressed_buffers = self.decompress_buffers()?; - self.build_inner(&decompressed_buffers, session) + self.build_inner(dtype, len, &decompressed_buffers, session) } // This is exposed to help non-CPU executors pass uncompressed buffer handles // to build the inner array. pub fn build_inner( &self, + dtype: &DType, + len: usize, buffer_handles: &[BufferHandle], session: &VortexSession, ) -> VortexResult { @@ -272,8 +266,8 @@ impl ZstdBuffersData { let children: Vec = self.slots.iter().flatten().cloned().collect(); inner_vtable.build( self.inner_encoding_id.clone(), - &self.dtype, - self.len, + dtype, + len, &self.inner_metadata, buffer_handles, &children.as_slice(), @@ -346,29 +340,15 @@ fn array_id_from_string(s: &str) -> ArrayId { impl VTable for ZstdBuffers { type ArrayData = ZstdBuffersData; - - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &ZstdBuffers - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ZstdBuffersData) -> usize { - array.len - } - - fn dtype(array: &ZstdBuffersData) -> &DType { - &array.dtype - } - - fn stats(array: &ZstdBuffersData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, _dtype: &DType, _len: usize) -> VortexResult<()> { + data.validate() } fn array_hash( @@ -433,36 +413,26 @@ impl VTable for ZstdBuffers { Ok(()) } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(ZstdBuffersMetadata { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some(ZstdBuffersMetadata { inner_encoding_id: array.inner_encoding_id.to_string(), inner_metadata: array.inner_metadata.clone(), uncompressed_sizes: array.uncompressed_sizes.clone(), buffer_alignments: array.buffer_alignments.clone(), - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.0.encode_to_vec())) + } + .encode_to_vec())) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata(ZstdBuffersMetadata::decode(bytes)?)) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = ZstdBuffersMetadata::decode(metadata)?; let compressed_buffers: Vec = buffers.to_vec(); let child_arrays: Vec> = (0..children.len()) @@ -470,15 +440,12 @@ impl VTable for ZstdBuffers { .collect::>>()?; let data = ZstdBuffersData { - inner_encoding_id: array_id_from_string(&metadata.0.inner_encoding_id), - inner_metadata: metadata.0.inner_metadata.clone(), + inner_encoding_id: array_id_from_string(&metadata.inner_encoding_id), + inner_metadata: metadata.inner_metadata.clone(), compressed_buffers, - uncompressed_sizes: metadata.0.uncompressed_sizes.clone(), - buffer_alignments: metadata.0.buffer_alignments.clone(), + uncompressed_sizes: metadata.uncompressed_sizes.clone(), + buffer_alignments: metadata.buffer_alignments.clone(), slots: child_arrays, - dtype: dtype.clone(), - len, - stats_set: Default::default(), }; data.validate()?; @@ -489,7 +456,7 @@ impl VTable for ZstdBuffers { fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { let session = ctx.session(); - let inner_array = array.decompress_and_build_inner(session)?; + let inner_array = ZstdBuffers::decompress_and_build_inner(&array, session)?; inner_array .execute::(ctx) .map(ExecutionResult::done) @@ -505,7 +472,11 @@ impl OperationsVTable for ZstdBuffers { // TODO(os): maybe we should not support scalar_at, it is really slow, and adding a cache // layer here is weird. Valid use of zstd buffers array would be by executing it first into // canonical - let inner_array = array.decompress_and_build_inner(&vortex_array::LEGACY_SESSION)?; + let inner_array = array.data().decompress_and_build_inner( + array.dtype(), + array.len(), + &vortex_array::LEGACY_SESSION, + )?; inner_array.scalar_at(index) } } @@ -514,11 +485,15 @@ impl ValidityVTable for ZstdBuffers { fn validity( array: ArrayView<'_, ZstdBuffers>, ) -> VortexResult { - if !array.dtype.is_nullable() { + if !array.dtype().is_nullable() { return Ok(vortex_array::validity::Validity::NonNullable); } - let inner_array = array.decompress_and_build_inner(&vortex_array::LEGACY_SESSION)?; + let inner_array = array.data().decompress_and_build_inner( + array.dtype(), + array.len(), + &vortex_array::LEGACY_SESSION, + )?; inner_array.validity() } } @@ -580,10 +555,10 @@ mod tests { #[case::empty_primitive(make_empty_primitive_array())] #[case::inlined_varbinview(make_inlined_varbinview_array())] fn test_roundtrip(#[case] input: ArrayRef) -> VortexResult<()> { - let compressed = ZstdBuffersData::compress(&input, 3)?; + let compressed = ZstdBuffers::compress(&input, 3)?; - assert_eq!(compressed.len, input.len()); - assert_eq!(&compressed.dtype, input.dtype()); + assert_eq!(compressed.len(), input.len()); + assert_eq!(compressed.dtype(), input.dtype()); let mut ctx = LEGACY_SESSION.create_execution_ctx(); let decompressed = compressed.into_array().execute::(&mut ctx)?; @@ -597,8 +572,7 @@ mod tests { let input = make_primitive_array(); input.statistics().set(Stat::Min, Precision::exact(0i32)); - let compressed = ZstdBuffersData::compress(&input, 3)?; - let compressed = ZstdBuffersArray::try_from_data(compressed)?; + let compressed = ZstdBuffers::compress(&input, 3)?; assert!(compressed.statistics().get(Stat::Min).is_some()); Ok(()) @@ -607,7 +581,7 @@ mod tests { #[test] fn test_validity_delegates_for_nullable_input() -> VortexResult<()> { let input = make_nullable_primitive_array(); - let compressed = ZstdBuffersData::compress(&input, 3)?.into_array(); + let compressed = ZstdBuffers::compress(&input, 3)?.into_array(); assert_eq!(compressed.all_valid()?, input.all_valid()?); assert_eq!(compressed.all_invalid()?, input.all_invalid()?); diff --git a/vortex-btrblocks/src/schemes/string.rs b/vortex-btrblocks/src/schemes/string.rs index 06ba6d03314..f222ce1623b 100644 --- a/vortex-btrblocks/src/schemes/string.rs +++ b/vortex-btrblocks/src/schemes/string.rs @@ -240,7 +240,7 @@ impl Scheme for ZstdBuffersScheme { let stats = data.string_stats(); Ok( - vortex_zstd::ZstdBuffersData::compress(&stats.source().clone().into_array(), 3)? + vortex_zstd::ZstdBuffers::compress(&stats.source().clone().into_array(), 3)? .into_array(), ) } diff --git a/vortex-cuda/src/hybrid_dispatch/mod.rs b/vortex-cuda/src/hybrid_dispatch/mod.rs index 06ac2c2738d..3832b9a48ad 100644 --- a/vortex-cuda/src/hybrid_dispatch/mod.rs +++ b/vortex-cuda/src/hybrid_dispatch/mod.rs @@ -235,7 +235,6 @@ mod tests { use vortex::array::session::ArraySessionExt; use vortex::encodings::fastlanes; use vortex::encodings::zstd::ZstdBuffers; - use vortex::encodings::zstd::ZstdBuffersData; let mut session = VortexSession::empty(); fastlanes::initialize(&mut session); @@ -256,7 +255,7 @@ mod tests { 0u32.into(), ) .vortex_expect("for"); - let vals = ZstdBuffersData::compress(&vals.into_array(), 3).vortex_expect("zstd"); + let vals = ZstdBuffers::compress(&vals.into_array(), 3).vortex_expect("zstd"); // codes = FoR(BitPacked) let codes = PrimitiveArray::new( diff --git a/vortex-cuda/src/kernel/encodings/zstd_buffers.rs b/vortex-cuda/src/kernel/encodings/zstd_buffers.rs index 1891ee2375c..d44b57615dc 100644 --- a/vortex-cuda/src/kernel/encodings/zstd_buffers.rs +++ b/vortex-cuda/src/kernel/encodings/zstd_buffers.rs @@ -55,7 +55,7 @@ async fn decode_zstd_buffers( let compressed_buffers = plan.compressed_buffers(); if compressed_buffers.is_empty() { - let inner_array = array.build_inner(&[], ctx.session())?; + let inner_array = ZstdBuffers::build_inner(&array, &[], ctx.session())?; return inner_array.execute_cuda(ctx).await; } @@ -158,7 +158,7 @@ async fn decode_zstd_buffers( let output_handle = BufferHandle::new_device(Arc::new(CudaDeviceBuffer::new(device_output))); let decompressed_buffers = plan.split_output_handle(&output_handle)?; - let inner_array = array.build_inner(&decompressed_buffers, ctx.session())?; + let inner_array = ZstdBuffers::build_inner(&array, &decompressed_buffers, ctx.session())?; inner_array.execute_cuda(ctx).await } @@ -223,7 +223,6 @@ mod tests { use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::VarBinViewArray; use vortex::array::assert_arrays_eq; - use vortex::encodings::zstd::ZstdBuffersData; use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::session::VortexSession; @@ -238,7 +237,7 @@ mod tests { .vortex_expect("failed to create execution context"); let input = PrimitiveArray::from_iter(0i64..1024).into_array(); - let compressed = ZstdBuffersData::compress(&input, 3)?; + let compressed = ZstdBuffers::compress(&input, 3)?; let cpu_result = compressed.clone().into_array().to_canonical()?; let gpu_result = ZstdBuffersExecutor @@ -265,7 +264,7 @@ mod tests { "baz", ]) .into_array(); - let compressed = ZstdBuffersData::compress(&input, 3)?; + let compressed = ZstdBuffers::compress(&input, 3)?; let cpu_result = compressed.clone().into_array().to_canonical()?; let gpu_result = ZstdBuffersExecutor From 8f761a508c0184618a9aacbfd9b1cd1e42a916d9 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Thu, 2 Apr 2026 21:28:33 -0400 Subject: [PATCH 03/15] Advanced frame parsing Signed-off-by: Nicholas Gates --- encodings/alp/src/alp/array.rs | 162 ++++++------ encodings/alp/src/alp/compress.rs | 7 +- encodings/alp/src/alp/compute/between.rs | 2 +- encodings/alp/src/alp/compute/cast.rs | 2 +- encodings/alp/src/alp/compute/filter.rs | 1 - encodings/alp/src/alp/decompress.rs | 6 +- encodings/alp/src/alp/ops.rs | 2 +- encodings/alp/src/alp_rd/array.rs | 233 +++++++++--------- encodings/alp/src/alp_rd/ops.rs | 3 +- encodings/bytebool/src/array.rs | 21 +- encodings/datetime-parts/src/array.rs | 20 +- .../src/decimal_byte_parts/compute/mod.rs | 1 - .../src/decimal_byte_parts/mod.rs | 21 +- .../fastlanes/src/bitpacking/vtable/mod.rs | 36 ++- encodings/fastlanes/src/delta/vtable/mod.rs | 24 +- encodings/fastlanes/src/for/vtable/mod.rs | 10 +- encodings/fastlanes/src/rle/array/mod.rs | 11 +- encodings/fastlanes/src/rle/vtable/mod.rs | 29 ++- encodings/fsst/src/array.rs | 64 ++--- encodings/parquet-variant/src/array.rs | 22 +- encodings/parquet-variant/src/vtable.rs | 10 +- encodings/pco/src/array.rs | 12 +- encodings/pco/src/test.rs | 4 +- encodings/runend/src/array.rs | 26 +- encodings/runend/src/arrow.rs | 4 +- encodings/sequence/src/array.rs | 40 +-- encodings/sparse/src/canonical.rs | 4 +- encodings/sparse/src/lib.rs | 12 +- encodings/zigzag/src/array.rs | 19 +- encodings/zstd/src/array.rs | 14 +- encodings/zstd/src/zstd_buffers.rs | 30 ++- vortex-array/src/array/typed.rs | 20 +- vortex-array/src/array/vtable/dyn_.rs | 8 +- vortex-array/src/arrays/bool/array.rs | 44 ++-- vortex-array/src/arrays/bool/vtable/mod.rs | 37 ++- vortex-array/src/arrays/chunked/array.rs | 31 ++- vortex-array/src/arrays/chunked/vtable/mod.rs | 31 +-- vortex-array/src/arrays/constant/array.rs | 34 +-- .../src/arrays/constant/vtable/mod.rs | 14 +- vortex-array/src/arrays/decimal/array.rs | 70 +++--- .../src/arrays/decimal/compute/between.rs | 2 +- vortex-array/src/arrays/decimal/vtable/mod.rs | 39 ++- vortex-array/src/arrays/dict/array.rs | 34 +-- vortex-array/src/arrays/dict/vtable/mod.rs | 13 +- .../src/arrays/dict/vtable/validity.rs | 1 - vortex-array/src/arrays/extension/array.rs | 23 +- .../src/arrays/extension/vtable/mod.rs | 20 +- vortex-array/src/arrays/filter/array.rs | 11 +- vortex-array/src/arrays/filter/vtable.rs | 24 +- .../src/arrays/fixed_size_list/array.rs | 70 +++--- .../src/arrays/fixed_size_list/vtable/mod.rs | 20 +- vortex-array/src/arrays/list/array.rs | 47 ++-- vortex-array/src/arrays/list/vtable/mod.rs | 20 +- vortex-array/src/arrays/listview/array.rs | 49 ++-- .../src/arrays/listview/conversion.rs | 4 +- .../src/arrays/listview/vtable/mod.rs | 20 +- .../src/arrays/listview/vtable/operations.rs | 2 +- vortex-array/src/arrays/masked/array.rs | 17 +- vortex-array/src/arrays/masked/vtable/mod.rs | 17 +- vortex-array/src/arrays/null/mod.rs | 40 +-- vortex-array/src/arrays/patched/array.rs | 12 +- vortex-array/src/arrays/patched/vtable/mod.rs | 18 +- .../src/arrays/primitive/array/mod.rs | 91 ++++--- .../src/arrays/primitive/compute/between.rs | 2 +- .../src/arrays/primitive/vtable/mod.rs | 27 +- vortex-array/src/arrays/scalar_fn/array.rs | 47 +--- vortex-array/src/arrays/scalar_fn/metadata.rs | 21 -- vortex-array/src/arrays/scalar_fn/mod.rs | 1 - vortex-array/src/arrays/scalar_fn/rules.rs | 6 +- .../src/arrays/scalar_fn/vtable/mod.rs | 34 ++- .../src/arrays/scalar_fn/vtable/operations.rs | 4 +- vortex-array/src/arrays/shared/array.rs | 14 +- vortex-array/src/arrays/shared/vtable.rs | 15 +- vortex-array/src/arrays/slice/array.rs | 9 +- vortex-array/src/arrays/slice/vtable.rs | 8 +- vortex-array/src/arrays/struct_/array.rs | 21 +- vortex-array/src/arrays/struct_/vtable/mod.rs | 1 - vortex-array/src/arrays/varbin/array.rs | 50 ++-- vortex-array/src/arrays/varbin/builder.rs | 2 - vortex-array/src/arrays/varbin/vtable/mod.rs | 1 - vortex-array/src/arrays/varbinview/array.rs | 39 ++- .../src/arrays/varbinview/vtable/mod.rs | 8 +- vortex-array/src/arrays/variant/mod.rs | 16 +- vortex-array/src/arrays/variant/vtable/mod.rs | 1 - vortex-array/src/scalar_fn/fns/select.rs | 4 +- vortex-array/src/serde.rs | 50 ++-- vortex-btrblocks/src/schemes/float.rs | 16 +- vortex-btrblocks/src/schemes/integer.rs | 23 +- vortex-btrblocks/src/schemes/rle.rs | 1 - vortex-cuda/src/kernel/encodings/bitpacked.rs | 2 +- vortex-cuda/src/layout.rs | 4 +- vortex-ipc/src/messages/decoder.rs | 6 +- vortex-layout/src/display.rs | 10 +- vortex-layout/src/layouts/flat/reader.rs | 6 +- vortex-python/src/arrays/py/array.rs | 16 ++ vortex-python/src/arrays/py/vtable.rs | 73 +----- vortex-python/src/serde/parts.rs | 12 +- vortex-tui/src/browse/app.rs | 4 +- vortex-tui/src/wasm.rs | 4 +- vortex-web/crate/src/wasm.rs | 15 +- 100 files changed, 1179 insertions(+), 1129 deletions(-) delete mode 100644 vortex-array/src/arrays/scalar_fn/metadata.rs diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index a5930c9b1ce..17d624f455a 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -4,19 +4,18 @@ use std::fmt::Debug; use std::hash::Hash; +use prost::Message; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; -use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; -use vortex_array::SerializeMetadata; use vortex_array::arrays::Primitive; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; @@ -26,7 +25,6 @@ use vortex_array::patches::PatchesMetadata; use vortex_array::require_child; use vortex_array::require_patches; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityChild; @@ -49,28 +47,15 @@ vtable!(ALP, ALP, ALPData); impl VTable for ALP { type ArrayData = ALPData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; - fn vtable(_array: &Self::ArrayData) -> &Self { - &ALP - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ALPData) -> usize { - array.encoded().len() - } - - fn dtype(array: &ALPData) -> &DType { - &array.dtype - } - - fn stats(array: &ALPData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &ALPData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate_against_outer(dtype, len) } fn array_hash(array: &ALPData, state: &mut H, precision: Precision) { @@ -97,41 +82,31 @@ impl VTable for ALP { None } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { let exponents = array.exponents(); - Ok(ProstMetadata(ALPMetadata { - exp_e: exponents.e as u32, - exp_f: exponents.f as u32, - patches: array - .patches() - .map(|p| p.to_metadata(array.len(), array.dtype())) - .transpose()?, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) - } - - fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata( - as DeserializeMetadata>::deserialize(bytes)?, + Ok(Some( + ALPMetadata { + exp_e: exponents.e as u32, + exp_f: exponents.f as u32, + patches: array + .patches() + .map(|p| p.to_metadata(array.len(), array.dtype())) + .transpose()?, + } + .encode_to_vec(), )) } - fn build( + fn deserialize( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = ALPMetadata::decode(metadata)?; let encoded_ptype = match &dtype { DType::Primitive(PType::F32, n) => DType::Primitive(PType::I32, *n), DType::Primitive(PType::F64, n) => DType::Primitive(PType::I64, *n), @@ -243,9 +218,7 @@ pub struct ALPData { pub(super) slots: Vec>, patch_offset: Option, patch_offset_within_chunk: Option, - dtype: DType, exponents: Exponents, - stats_set: ArrayStats, } #[derive(Clone, Debug)] @@ -266,7 +239,7 @@ pub struct ALPMetadata { } impl ALPData { - fn validate( + fn validate_components( encoded: &ArrayRef, exponents: Exponents, patches: Option<&Patches>, @@ -317,6 +290,38 @@ impl ALPData { Ok(()) } + fn logical_dtype(encoded: &ArrayRef) -> VortexResult { + match encoded.dtype() { + DType::Primitive(PType::I32, nullability) => { + Ok(DType::Primitive(PType::F32, *nullability)) + } + DType::Primitive(PType::I64, nullability) => { + Ok(DType::Primitive(PType::F64, *nullability)) + } + _ => vortex_bail!("ALP encoded ints have invalid DType {}", encoded.dtype(),), + } + } + + fn validate_against_outer(&self, dtype: &DType, len: usize) -> VortexResult<()> { + let patches = self.patches(); + let logical_dtype = Self::logical_dtype(self.encoded())?; + Self::validate_components(self.encoded(), self.exponents, patches.as_ref())?; + + vortex_ensure!( + self.encoded().len() == len, + "ALP encoded len {} != outer len {len}", + self.encoded().len(), + ); + vortex_ensure!( + &logical_dtype == dtype, + "ALP dtype {} does not match encoded logical dtype {}", + dtype, + logical_dtype, + ); + + Ok(()) + } + /// Validate that any patches provided are valid for the ALPArray. fn validate_patches(patches: &Patches, encoded: &ArrayRef) -> VortexResult<()> { vortex_ensure!( @@ -402,13 +407,7 @@ impl ALPData { exponents: Exponents, patches: Option, ) -> VortexResult { - Self::validate(&encoded, exponents, patches.as_ref())?; - - let dtype = match encoded.dtype() { - DType::Primitive(PType::I32, nullability) => DType::Primitive(PType::F32, *nullability), - DType::Primitive(PType::I64, nullability) => DType::Primitive(PType::F64, *nullability), - _ => unreachable!(), - }; + Self::validate_components(&encoded, exponents, patches.as_ref())?; let slots = Self::make_slots(&encoded, &patches); let (patch_offset, patch_offset_within_chunk) = match &patches { @@ -417,12 +416,10 @@ impl ALPData { }; Ok(Self { - dtype, slots, exponents, patch_offset, patch_offset_within_chunk, - stats_set: Default::default(), }) } @@ -434,7 +431,6 @@ impl ALPData { encoded: ArrayRef, exponents: Exponents, patches: Option, - dtype: DType, ) -> Self { let slots = Self::make_slots(&encoded, &patches); let (patch_offset, patch_offset_within_chunk) = match &patches { @@ -443,12 +439,10 @@ impl ALPData { }; Self { - dtype, slots, exponents, patch_offset, patch_offset_within_chunk, - stats_set: Default::default(), } } } @@ -456,8 +450,15 @@ impl ALPData { /// Constructors for [`ALPArray`]. impl ALP { pub fn new(encoded: ArrayRef, exponents: Exponents, patches: Option) -> ALPArray { - Array::try_from_data(ALPData::new(encoded, exponents, patches)) - .vortex_expect("ALPData is always valid") + let dtype = ALPData::logical_dtype(&encoded).vortex_expect("ALP encoded dtype"); + let len = encoded.len(); + Array::try_from_parts(ArrayParts::new( + ALP, + dtype, + len, + ALPData::new(encoded, exponents, patches), + )) + .vortex_expect("ALPData is always valid") } pub fn try_new( @@ -465,7 +466,10 @@ impl ALP { exponents: Exponents, patches: Option, ) -> VortexResult { - Array::try_from_data(ALPData::try_new(encoded, exponents, patches)?) + let dtype = ALPData::logical_dtype(&encoded)?; + let len = encoded.len(); + let data = ALPData::try_new(encoded, exponents, patches)?; + Array::try_from_parts(ArrayParts::new(ALP, dtype, len, data)) } /// # Safety @@ -474,29 +478,17 @@ impl ALP { encoded: ArrayRef, exponents: Exponents, patches: Option, - dtype: DType, ) -> ALPArray { - Array::try_from_data(unsafe { ALPData::new_unchecked(encoded, exponents, patches, dtype) }) - .vortex_expect("ALPData is always valid") + let dtype = ALPData::logical_dtype(&encoded).vortex_expect("ALP encoded dtype"); + let len = encoded.len(); + Array::try_from_parts(ArrayParts::new(ALP, dtype, len, unsafe { + ALPData::new_unchecked(encoded, exponents, patches) + })) + .vortex_expect("ALPData is always valid") } } impl ALPData { - /// Returns the number of elements in the array. - pub fn len(&self) -> usize { - self.encoded().len() - } - - /// Returns `true` if the array contains no elements. - pub fn is_empty(&self) -> bool { - self.encoded().len() == 0 - } - - /// Returns the logical data type of the array. - pub fn dtype(&self) -> &DType { - &self.dtype - } - fn make_slots(encoded: &ArrayRef, patches: &Option) -> Vec> { let (patch_indices, patch_values, patch_chunk_offsets) = match patches { Some(p) => ( @@ -514,10 +506,6 @@ impl ALPData { ] } - pub fn ptype(&self) -> PType { - self.dtype.as_ptype() - } - pub fn encoded(&self) -> &ArrayRef { self.slots[ENCODED_SLOT] .as_ref() @@ -555,12 +543,12 @@ impl ALPData { /// Consumes the array and returns its parts. #[inline] - pub fn into_parts(mut self) -> (ArrayRef, Exponents, Option, DType) { + pub fn into_parts(mut self) -> (ArrayRef, Exponents, Option) { let patches = self.patches(); let encoded = self.slots[ENCODED_SLOT] .take() .vortex_expect("ALPArray encoded slot"); - (encoded, self.exponents, patches, self.dtype) + (encoded, self.exponents, patches) } } diff --git a/encodings/alp/src/alp/compress.rs b/encodings/alp/src/alp/compress.rs index e8992c7a2a4..668799f8984 100644 --- a/encodings/alp/src/alp/compress.rs +++ b/encodings/alp/src/alp/compress.rs @@ -48,12 +48,7 @@ pub fn alp_encode(parray: &PrimitiveArray, exponents: Option) -> Vort // SAFETY: alp_encode_components_typed must return well-formed components unsafe { - Ok(ALP::new_unchecked( - encoded, - exponents, - patches, - parray.dtype().clone(), - )) + Ok(ALP::new_unchecked(encoded, exponents, patches)) } } diff --git a/encodings/alp/src/alp/compute/between.rs b/encodings/alp/src/alp/compute/between.rs index ff0b4da8943..efaaf747768 100644 --- a/encodings/alp/src/alp/compute/between.rs +++ b/encodings/alp/src/alp/compute/between.rs @@ -38,7 +38,7 @@ impl BetweenReduce for ALP { let nullability = array.dtype().nullability() | lower.dtype().nullability() | upper.dtype().nullability(); - match_each_alp_float_ptype!(array.ptype(), |F| { + match_each_alp_float_ptype!(array.dtype().as_ptype(), |F| { between_impl::( array, F::try_from(&lower)?, diff --git a/encodings/alp/src/alp/compute/cast.rs b/encodings/alp/src/alp/compute/cast.rs index ebca5e3fe0f..d07b2a7d128 100644 --- a/encodings/alp/src/alp/compute/cast.rs +++ b/encodings/alp/src/alp/compute/cast.rs @@ -45,7 +45,7 @@ impl CastReduce for ALP { // SAFETY: casting nullability doesn't alter the invariants unsafe { Ok(Some( - ALP::new_unchecked(new_encoded, array.exponents(), new_patches, dtype.clone()) + ALP::new_unchecked(new_encoded, array.exponents(), new_patches) .into_array(), )) } diff --git a/encodings/alp/src/alp/compute/filter.rs b/encodings/alp/src/alp/compute/filter.rs index a4e3c4371d9..3daa6d53d7b 100644 --- a/encodings/alp/src/alp/compute/filter.rs +++ b/encodings/alp/src/alp/compute/filter.rs @@ -30,7 +30,6 @@ impl FilterKernel for ALP { array.encoded().filter(mask.clone())?, array.exponents(), patches, - array.dtype().clone(), ) .into_array(), )) diff --git a/encodings/alp/src/alp/decompress.rs b/encodings/alp/src/alp/decompress.rs index 0e7c1a5e7a0..e118b9705cd 100644 --- a/encodings/alp/src/alp/decompress.rs +++ b/encodings/alp/src/alp/decompress.rs @@ -27,7 +27,8 @@ pub fn decompress_into_array( array: ALPArray, ctx: &mut ExecutionCtx, ) -> VortexResult { - let (encoded, exponents, patches, dtype) = array.into_data().into_parts(); + let dtype = array.dtype().clone(); + let (encoded, exponents, patches) = array.into_data().into_parts(); if let Some(ref patches) = patches && let Some(chunk_offsets) = patches.chunk_offsets() { @@ -59,7 +60,8 @@ pub fn decompress_into_array( /// /// A `PrimitiveArray` containing the decompressed floating-point values with all patches applied. pub fn execute_decompress(array: ALPArray, ctx: &mut ExecutionCtx) -> VortexResult { - let (encoded, exponents, patches, dtype) = array.into_data().into_parts(); + let dtype = array.dtype().clone(); + let (encoded, exponents, patches) = array.into_data().into_parts(); if let Some(ref patches) = patches && let Some(chunk_offsets) = patches.chunk_offsets() { diff --git a/encodings/alp/src/alp/ops.rs b/encodings/alp/src/alp/ops.rs index 9538e3c60a1..25cf19a3e55 100644 --- a/encodings/alp/src/alp/ops.rs +++ b/encodings/alp/src/alp/ops.rs @@ -26,7 +26,7 @@ impl OperationsVTable for ALP { let encoded_val = array.encoded().scalar_at(index)?; - Ok(match_each_alp_float_ptype!(array.ptype(), |T| { + Ok(match_each_alp_float_ptype!(array.dtype().as_ptype(), |T| { let encoded_val: ::ALPInt = (&encoded_val).try_into().vortex_expect("invalid ALPInt"); Scalar::primitive( diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index eca56a97a37..a785fc28552 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -5,19 +5,18 @@ use std::fmt::Debug; use std::hash::Hash; use itertools::Itertools; +use prost::Message; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; -use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; -use vortex_array::SerializeMetadata; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; @@ -29,7 +28,6 @@ use vortex_array::patches::PatchesMetadata; use vortex_array::require_child; use vortex_array::require_patches; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; @@ -67,28 +65,15 @@ pub struct ALPRDMetadata { impl VTable for ALPRD { type ArrayData = ALPRDData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; - fn vtable(_array: &Self::ArrayData) -> &Self { - &ALPRD - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ALPRDData) -> usize { - array.left_parts().len() - } - - fn dtype(array: &ALPRDData) -> &DType { - &array.dtype - } - - fn stats(array: &ALPRDData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &ALPRDData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate_against_outer(dtype, len) } fn array_hash(array: &ALPRDData, state: &mut H, precision: Precision) { @@ -123,48 +108,38 @@ impl VTable for ALPRD { None } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { let dict = array .left_parts_dictionary() .iter() .map(|&i| i as u32) .collect::>(); - Ok(ProstMetadata(ALPRDMetadata { - right_bit_width: array.right_bit_width() as u32, - dict_len: array.left_parts_dictionary().len() as u32, - dict, - left_parts_ptype: array.left_parts().dtype().as_ptype() as i32, - patches: array - .left_parts_patches() - .map(|p| p.to_metadata(array.len(), array.left_parts().dtype())) - .transpose()?, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) - } - - fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata( - as DeserializeMetadata>::deserialize(bytes)?, + Ok(Some( + ALPRDMetadata { + right_bit_width: array.right_bit_width() as u32, + dict_len: array.left_parts_dictionary().len() as u32, + dict, + left_parts_ptype: array.left_parts().dtype().as_ptype() as i32, + patches: array + .left_parts_patches() + .map(|p| p.to_metadata(array.len(), array.left_parts().dtype())) + .transpose()?, + } + .encode_to_vec(), )) } - fn build( + fn deserialize( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = ALPRDMetadata::decode(metadata)?; if children.len() < 2 { vortex_bail!( "Expected at least 2 children for ALPRD encoding, found {}", @@ -172,10 +147,10 @@ impl VTable for ALPRD { ); } - let left_parts_dtype = DType::Primitive(metadata.0.left_parts_ptype(), dtype.nullability()); + let left_parts_dtype = DType::Primitive(metadata.left_parts_ptype(), dtype.nullability()); let left_parts = children.get(0, &left_parts_dtype, len)?; - let left_parts_dictionary: Buffer = metadata.0.dict.as_slice() - [0..metadata.0.dict_len as usize] + let left_parts_dictionary: Buffer = metadata.dict.as_slice() + [0..metadata.dict_len as usize] .iter() .map(|&i| { u16::try_from(i) @@ -195,7 +170,6 @@ impl VTable for ALPRD { let right_parts = children.get(1, &right_parts_dtype, len)?; let left_parts_patches = metadata - .0 .patches .map(|p| { let indices = children.get(2, &p.indices_dtype()?, p.len()?)?; @@ -217,10 +191,10 @@ impl VTable for ALPRD { left_parts, left_parts_dictionary, right_parts, - u8::try_from(metadata.0.right_bit_width).map_err(|_| { + u8::try_from(metadata.right_bit_width).map_err(|_| { vortex_err!( "right_bit_width {} out of u8 range", - metadata.0.right_bit_width + metadata.right_bit_width ) })?, left_parts_patches, @@ -276,14 +250,13 @@ impl VTable for ALPRD { LP_PATCH_CHUNK_OFFSETS_SLOT ); + let dtype = array.dtype().clone(); let right_bit_width = array.right_bit_width(); let ALPRDArrayParts { left_parts, right_parts, left_parts_dictionary, left_parts_patches, - dtype, - .. } = array.into_data().into_parts(); let ptype = dtype.as_ptype(); @@ -368,17 +341,14 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = [ #[derive(Clone, Debug)] pub struct ALPRDData { - dtype: DType, slots: Vec>, left_parts_patches: Option, left_parts_dictionary: Buffer, right_bit_width: u8, - stats_set: ArrayStats, } #[derive(Clone, Debug)] pub struct ALPRDArrayParts { - pub dtype: DType, pub left_parts: ArrayRef, pub left_parts_patches: Option, pub left_parts_dictionary: Buffer, @@ -399,14 +369,17 @@ impl ALPRD { right_bit_width: u8, left_parts_patches: Option, ) -> VortexResult { - Array::try_from_data(ALPRDData::try_new( + let len = left_parts.len(); + let logical_dtype = dtype.clone(); + let data = ALPRDData::try_new( dtype, left_parts, left_parts_dictionary, right_parts, right_bit_width, left_parts_patches, - )?) + )?; + Array::try_from_parts(ArrayParts::new(ALPRD, logical_dtype, len, data)) } /// # Safety @@ -419,7 +392,9 @@ impl ALPRD { right_bit_width: u8, left_parts_patches: Option, ) -> ALPRDArray { - Array::try_from_data(unsafe { + let len = left_parts.len(); + let logical_dtype = dtype.clone(); + let data = unsafe { ALPRDData::new_unchecked( dtype, left_parts, @@ -428,33 +403,34 @@ impl ALPRD { right_bit_width, left_parts_patches, ) - }) - .vortex_expect("ALPRDData is always valid") + }; + Array::try_from_parts(ArrayParts::new(ALPRD, logical_dtype, len, data)) + .vortex_expect("ALPRDData is always valid") } } impl ALPRDData { - /// Build a new `ALPRDArray` from components. - pub fn try_new( - dtype: DType, - left_parts: ArrayRef, - left_parts_dictionary: Buffer, - right_parts: ArrayRef, - right_bit_width: u8, - left_parts_patches: Option, - ) -> VortexResult { + fn validate_parts( + dtype: &DType, + len: usize, + left_parts: &ArrayRef, + right_parts: &ArrayRef, + left_parts_patches: Option<&Patches>, + ) -> VortexResult<()> { if !dtype.is_float() { vortex_bail!("ALPRDArray given invalid DType ({dtype})"); } - let len = left_parts.len(); - if right_parts.len() != len { - vortex_bail!( - "left_parts (len {}) and right_parts (len {}) must be of same length", - len, - right_parts.len() - ); - } + vortex_ensure!( + left_parts.len() == len, + "left_parts len {} != outer len {len}", + left_parts.len(), + ); + vortex_ensure!( + right_parts.len() == len, + "right_parts len {} != outer len {len}", + right_parts.len(), + ); if !left_parts.dtype().is_unsigned_int() { vortex_bail!("left_parts dtype must be uint"); @@ -468,11 +444,64 @@ impl ALPRDData { ); } - // we enforce right_parts to be non-nullable uint - if !right_parts.dtype().is_unsigned_int() || right_parts.dtype().is_nullable() { - vortex_bail!(MismatchedTypes: "non-nullable uint", right_parts.dtype()); + let expected_right_parts_dtype = match dtype { + DType::Primitive(PType::F32, _) => { + DType::Primitive(PType::U32, Nullability::NonNullable) + } + DType::Primitive(PType::F64, _) => { + DType::Primitive(PType::U64, Nullability::NonNullable) + } + _ => vortex_bail!("Expected f32 or f64 dtype, got {:?}", dtype), + }; + vortex_ensure!( + right_parts.dtype() == &expected_right_parts_dtype, + "right_parts dtype {} does not match expected {}", + right_parts.dtype(), + expected_right_parts_dtype, + ); + + if let Some(patches) = left_parts_patches { + vortex_ensure!( + patches.array_len() == len, + "patches array_len {} != outer len {len}", + patches.array_len(), + ); + vortex_ensure!( + patches.dtype() == left_parts.dtype(), + "patches dtype {} does not match left_parts dtype {}", + patches.dtype(), + left_parts.dtype(), + ); + vortex_ensure!( + patches.values().all_valid()?, + "patches must be all valid: {}", + patches.values() + ); } + Ok(()) + } + + fn validate_against_outer(&self, dtype: &DType, len: usize) -> VortexResult<()> { + Self::validate_parts( + dtype, + len, + self.left_parts(), + self.right_parts(), + self.left_parts_patches.as_ref(), + ) + } + + /// Build a new `ALPRDArray` from components. + pub fn try_new( + dtype: DType, + left_parts: ArrayRef, + left_parts_dictionary: Buffer, + right_parts: ArrayRef, + right_bit_width: u8, + left_parts_patches: Option, + ) -> VortexResult { + let len = left_parts.len(); let left_parts_patches = left_parts_patches .map(|patches| { if !patches.values().all_valid()? { @@ -488,22 +517,28 @@ impl ALPRDData { }) .transpose()?; + Self::validate_parts( + &dtype, + len, + &left_parts, + &right_parts, + left_parts_patches.as_ref(), + )?; + let slots = Self::make_slots(&left_parts, &right_parts, &left_parts_patches); Ok(Self { - dtype, slots, left_parts_dictionary, right_bit_width, left_parts_patches, - stats_set: Default::default(), }) } /// Build a new `ALPRDArray` from components. This does not perform any validation, and instead /// it constructs it from parts. pub(crate) unsafe fn new_unchecked( - dtype: DType, + _dtype: DType, left_parts: ArrayRef, left_parts_dictionary: Buffer, right_parts: ArrayRef, @@ -513,12 +548,10 @@ impl ALPRDData { let slots = Self::make_slots(&left_parts, &right_parts, &left_parts_patches); Self { - dtype, slots, left_parts_patches, left_parts_dictionary, right_bit_width, - stats_set: Default::default(), } } @@ -553,7 +586,6 @@ impl ALPRDData { .take() .vortex_expect("ALPRDArray right_parts slot"); ALPRDArrayParts { - dtype: self.dtype, left_parts, left_parts_patches: self.left_parts_patches, left_parts_dictionary: self.left_parts_dictionary, @@ -561,29 +593,6 @@ impl ALPRDData { } } - /// Returns the number of elements in the array. - pub fn len(&self) -> usize { - self.left_parts().len() - } - - /// Returns `true` if the array contains no elements. - pub fn is_empty(&self) -> bool { - self.left_parts().len() == 0 - } - - /// Returns the logical data type of the array. - pub fn dtype(&self) -> &DType { - &self.dtype - } - - /// Returns true if logical type of the array values is f32. - /// - /// Returns false if the logical type of the array values is f64. - #[inline] - pub fn is_f32(&self) -> bool { - matches!(&self.dtype, DType::Primitive(PType::F32, _)) - } - /// The leftmost (most significant) bits of the floating point values stored in the array. /// /// These are bit-packed and dictionary encoded, and cannot directly be interpreted without diff --git a/encodings/alp/src/alp_rd/ops.rs b/encodings/alp/src/alp_rd/ops.rs index 63dc3fc5d89..6f77fdbd843 100644 --- a/encodings/alp/src/alp_rd/ops.rs +++ b/encodings/alp/src/alp_rd/ops.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; +use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; use vortex_array::vtable::OperationsVTable; use vortex_error::VortexExpect; @@ -39,7 +40,7 @@ impl OperationsVTable for ALPRD { }; // combine left and right values - Ok(if array.is_f32() { + Ok(if array.dtype().as_ptype() == PType::F32 { let right: u32 = array .right_parts() .scalar_at(index)? diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index 2e04a966943..aadb8405556 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -4,10 +4,10 @@ use std::fmt::Debug; use vortex_array::Array; -use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::EmptyMetadata; @@ -185,7 +185,7 @@ impl ByteBool { let dtype = DType::Bool(validity.nullability()); let data = ByteBoolData::new(buffer, validity); let len = data.len(); - Array::try_from_parts(ArrayNew::new(ByteBool, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(ByteBool, dtype, len, data)) .vortex_expect("ByteBoolData is always valid") } @@ -194,7 +194,7 @@ impl ByteBool { let data = ByteBoolData::from_vec(data, validity); let dtype = DType::Bool(data.validity.nullability()); let len = data.len(); - Array::try_from_parts(ArrayNew::new(ByteBool, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(ByteBool, dtype, len, data)) .vortex_expect("ByteBoolData is always valid") } } @@ -207,8 +207,15 @@ impl ByteBoolData { len: usize, ) -> VortexResult<()> { let expected_dtype = DType::Bool(validity.nullability()); - vortex_ensure!(dtype == &expected_dtype, "expected dtype {expected_dtype}, got {dtype}"); - vortex_ensure!(buffer.len() == len, "expected len {len}, got {}", buffer.len()); + vortex_ensure!( + dtype == &expected_dtype, + "expected dtype {expected_dtype}, got {dtype}" + ); + vortex_ensure!( + buffer.len() == len, + "expected len {len}, got {}", + buffer.len() + ); if let Some(vlen) = validity.maybe_len() { vortex_ensure!(vlen == len, "expected validity len {len}, got {vlen}"); } @@ -312,8 +319,8 @@ mod tests { use vortex_array::ArrayContext; use vortex_array::IntoArray; use vortex_array::assert_arrays_eq; - use vortex_array::serde::ArrayParts; use vortex_array::serde::SerializeOptions; + use vortex_array::serde::SerializedArray; use vortex_array::session::ArraySession; use vortex_array::session::ArraySessionExt; use vortex_buffer::ByteBufferMut; @@ -382,7 +389,7 @@ mod tests { concat.extend_from_slice(buf.as_ref()); } - let parts = ArrayParts::try_from(concat.freeze()).unwrap(); + let parts = SerializedArray::try_from(concat.freeze()).unwrap(); let decoded = parts .decode(&dtype, len, &ReadContext::new(ctx.to_ids()), &session) .unwrap(); diff --git a/encodings/datetime-parts/src/array.rs b/encodings/datetime-parts/src/array.rs index 841586aa0b8..2214f1c505f 100644 --- a/encodings/datetime-parts/src/array.rs +++ b/encodings/datetime-parts/src/array.rs @@ -4,10 +4,10 @@ use std::fmt::Debug; use vortex_array::Array; -use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::DeserializeMetadata; @@ -118,12 +118,14 @@ impl VTable for DateTimeParts { } fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some(ProstMetadata(DateTimePartsMetadata { - days_ptype: PType::try_from(array.days().dtype())? as i32, - seconds_ptype: PType::try_from(array.seconds().dtype())? as i32, - subseconds_ptype: PType::try_from(array.subseconds().dtype())? as i32, - }) - .serialize())) + Ok(Some( + ProstMetadata(DateTimePartsMetadata { + days_ptype: PType::try_from(array.days().dtype())? as i32, + seconds_ptype: PType::try_from(array.seconds().dtype())? as i32, + subseconds_ptype: PType::try_from(array.subseconds().dtype())? as i32, + }) + .serialize(), + )) } fn deserialize( @@ -242,7 +244,7 @@ impl DateTimeParts { ) -> VortexResult { let data = DateTimePartsData::try_new(dtype.clone(), days, seconds, subseconds)?; let len = data.len(); - Array::try_from_parts(ArrayNew::new(DateTimeParts, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(DateTimeParts, dtype, len, data)) } /// Construct a [`DateTimePartsArray`] from a [`TemporalArray`]. @@ -250,7 +252,7 @@ impl DateTimeParts { let dtype = temporal.dtype().clone(); let data = DateTimePartsData::try_from(temporal)?; let len = data.len(); - Array::try_from_parts(ArrayNew::new(DateTimeParts, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(DateTimeParts, dtype, len, data)) } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mod.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mod.rs index ef271694aea..35da5be40b6 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mod.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mod.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use super::DecimalBytePartsData; mod cast; mod compare; mod filter; diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs index ffea1d4ce41..d6fabe60737 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::Array; -use vortex_array::ArrayNew; +use vortex_array::ArrayParts; use vortex_array::ArrayView; pub(crate) mod compute; mod rules; @@ -101,11 +101,13 @@ impl VTable for DecimalByteParts { } fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some(ProstMetadata(DecimalBytesPartsMetadata { - zeroth_child_ptype: PType::try_from(array.msp().dtype())? as i32, - lower_part_count: 0, - }) - .serialize())) + Ok(Some( + ProstMetadata(DecimalBytesPartsMetadata { + zeroth_child_ptype: PType::try_from(array.msp().dtype())? as i32, + lower_part_count: 0, + }) + .serialize(), + )) } fn deserialize( @@ -212,7 +214,10 @@ impl DecimalBytePartsData { } let expected_dtype = DType::Decimal(decimal_dtype, msp.dtype().nullability()); - vortex_ensure!(dtype == &expected_dtype, "expected dtype {expected_dtype}, got {dtype}"); + vortex_ensure!( + dtype == &expected_dtype, + "expected dtype {expected_dtype}, got {dtype}" + ); vortex_ensure!(msp.len() == len, "expected len {len}, got {}", msp.len()); Ok(()) } @@ -257,7 +262,7 @@ impl DecimalByteParts { let data = DecimalBytePartsData::try_new(msp, decimal_dtype)?; let dtype = DType::Decimal(decimal_dtype, data.msp().dtype().nullability()); let len = data.len(); - Array::try_from_parts(ArrayNew::new(DecimalByteParts, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(DecimalByteParts, dtype, len, data)) } } diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs index 7a7865dd85a..38e9239732b 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs @@ -6,10 +6,10 @@ use std::hash::Hash; use prost::Message; use vortex_array::AnyCanonical; use vortex_array::Array; -use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; @@ -37,6 +37,7 @@ use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use crate::BitPackedArrayParts; use crate::BitPackedData; use crate::bitpack_decompress::unpack_array; use crate::bitpack_decompress::unpack_into_primitive_builder; @@ -114,7 +115,11 @@ impl VTable for BitPacked { &other.slots[PATCH_CHUNK_OFFSETS_SLOT], precision, ) - && option_array_eq(&array.slots[VALIDITY_SLOT], &other.slots[VALIDITY_SLOT], precision) + && option_array_eq( + &array.slots[VALIDITY_SLOT], + &other.slots[VALIDITY_SLOT], + precision, + ) && array.patch_offset == other.patch_offset && array.patch_offset_within_chunk == other.patch_offset_within_chunk } @@ -174,14 +179,14 @@ impl VTable for BitPacked { fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some( BitPackedMetadata { - bit_width: array.bit_width() as u32, - offset: array.offset() as u32, - patches: array - .patches(array.len()) - .map(|p| p.to_metadata(array.len(), array.dtype())) - .transpose()?, - } - .encode_to_vec(), + bit_width: array.bit_width() as u32, + offset: array.offset() as u32, + patches: array + .patches(array.len()) + .map(|p| p.to_metadata(array.len(), array.dtype())) + .transpose()?, + } + .encode_to_vec(), )) } @@ -321,8 +326,15 @@ impl BitPacked { offset: u16, ) -> VortexResult { let dtype = DType::Primitive(ptype, validity.nullability()); - let data = BitPackedData::try_new(packed, ptype, validity, patches, bit_width, len, offset)?; - Array::try_from_parts(ArrayNew::new(BitPacked, dtype, len, data)) + let data = + BitPackedData::try_new(packed, ptype, validity, patches, bit_width, len, offset)?; + Array::try_from_parts(ArrayParts::new(BitPacked, dtype, len, data)) + } + + pub fn into_parts(array: BitPackedArray) -> BitPackedArrayParts { + let len = array.len(); + let nullability = array.dtype().nullability(); + array.into_data().into_parts(len, nullability) } /// Encode an array into a bitpacked representation with the given bit width. diff --git a/encodings/fastlanes/src/delta/vtable/mod.rs b/encodings/fastlanes/src/delta/vtable/mod.rs index 137c3ff202f..c4bedc8d9a0 100644 --- a/encodings/fastlanes/src/delta/vtable/mod.rs +++ b/encodings/fastlanes/src/delta/vtable/mod.rs @@ -6,10 +6,10 @@ use std::hash::Hash; use fastlanes::FastLanes; use prost::Message; use vortex_array::Array; -use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; @@ -119,10 +119,10 @@ impl VTable for Delta { fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some( DeltaMetadata { - deltas_len: array.deltas().len() as u64, - offset: array.offset() as u32, - } - .encode_to_vec(), + deltas_len: array.deltas().len() as u64, + offset: array.offset() as u32, + } + .encode_to_vec(), )) } @@ -135,8 +135,16 @@ impl VTable for Delta { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - vortex_ensure!(buffers.is_empty(), "DeltaArray expects 0 buffers, got {}", buffers.len()); - vortex_ensure!(children.len() == 2, "DeltaArray expects 2 children, got {}", children.len()); + vortex_ensure!( + buffers.is_empty(), + "DeltaArray expects 0 buffers, got {}", + buffers.len() + ); + vortex_ensure!( + children.len() == 2, + "DeltaArray expects 2 children, got {}", + children.len() + ); let metadata = DeltaMetadata::decode(metadata)?; let ptype = PType::try_from(dtype)?; let lanes = match_each_unsigned_integer_ptype!(ptype, |T| { ::LANES }); @@ -175,7 +183,7 @@ impl Delta { ) -> VortexResult { let dtype = bases.dtype().with_nullability(deltas.dtype().nullability()); let data = DeltaData::try_new(bases, deltas, offset, len)?; - Array::try_from_parts(ArrayNew::new(Delta, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Delta, dtype, len, data)) } /// Compress a primitive array using Delta encoding. diff --git a/encodings/fastlanes/src/for/vtable/mod.rs b/encodings/fastlanes/src/for/vtable/mod.rs index 2af970af65b..5b6c4f199b6 100644 --- a/encodings/fastlanes/src/for/vtable/mod.rs +++ b/encodings/fastlanes/src/for/vtable/mod.rs @@ -5,10 +5,10 @@ use std::fmt::Debug; use std::hash::Hash; use vortex_array::Array; -use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; @@ -116,7 +116,11 @@ impl VTable for FoR { children: &dyn ArrayChildren, session: &VortexSession, ) -> VortexResult { - vortex_ensure!(buffers.is_empty(), "FoRArray expects 0 buffers, got {}", buffers.len()); + vortex_ensure!( + buffers.is_empty(), + "FoRArray expects 0 buffers, got {}", + buffers.len() + ); if children.len() != 1 { vortex_bail!( "Expected 1 child for FoR encoding, found {}", @@ -168,7 +172,7 @@ impl FoR { let reference = reference.cast(&dtype)?; let len = encoded.len(); let data = FoRData::try_new(encoded, reference)?; - Array::try_from_parts(ArrayNew::new(FoR, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(FoR, dtype, len, data)) } /// Encode a primitive array using Frame of Reference encoding. diff --git a/encodings/fastlanes/src/rle/array/mod.rs b/encodings/fastlanes/src/rle/array/mod.rs index f9a292e83de..7d0459d85f8 100644 --- a/encodings/fastlanes/src/rle/array/mod.rs +++ b/encodings/fastlanes/src/rle/array/mod.rs @@ -83,7 +83,10 @@ impl RLEData { ); vortex_ensure!( - matches!(values.dtype(), DType::Primitive(_, Nullability::NonNullable)), + matches!( + values.dtype(), + DType::Primitive(_, Nullability::NonNullable) + ), "RLE values must be a non-nullable primitive type, got {}", values.dtype() ); @@ -234,8 +237,8 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::serde::ArrayParts; use vortex_array::serde::SerializeOptions; + use vortex_array::serde::SerializedArray; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::ByteBufferMut; @@ -466,7 +469,7 @@ mod tests { } let concat = concat.freeze(); - let parts = ArrayParts::try_from(concat).unwrap(); + let parts = SerializedArray::try_from(concat).unwrap(); let decoded = parts .decode( &DType::Primitive(PType::U32, Nullability::NonNullable), @@ -512,7 +515,7 @@ mod tests { } let concat = concat.freeze(); - let parts = ArrayParts::try_from(concat).unwrap(); + let parts = SerializedArray::try_from(concat).unwrap(); let decoded = parts .decode( sliced.dtype(), diff --git a/encodings/fastlanes/src/rle/vtable/mod.rs b/encodings/fastlanes/src/rle/vtable/mod.rs index b449facaa52..8dc93872ba2 100644 --- a/encodings/fastlanes/src/rle/vtable/mod.rs +++ b/encodings/fastlanes/src/rle/vtable/mod.rs @@ -5,10 +5,10 @@ use std::hash::Hash; use prost::Message; use vortex_array::Array; -use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; @@ -128,14 +128,15 @@ impl VTable for RLE { fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some( RLEMetadata { - values_len: array.values().len() as u64, - indices_len: array.indices().len() as u64, - indices_ptype: PType::try_from(array.indices().dtype())? as i32, - values_idx_offsets_len: array.values_idx_offsets().len() as u64, - values_idx_offsets_ptype: PType::try_from(array.values_idx_offsets().dtype())? as i32, - offset: array.offset() as u64, - } - .encode_to_vec(), + values_len: array.values().len() as u64, + indices_len: array.indices().len() as u64, + indices_ptype: PType::try_from(array.indices().dtype())? as i32, + values_idx_offsets_len: array.values_idx_offsets().len() as u64, + values_idx_offsets_ptype: PType::try_from(array.values_idx_offsets().dtype())? + as i32, + offset: array.offset() as u64, + } + .encode_to_vec(), )) } @@ -148,7 +149,11 @@ impl VTable for RLE { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - vortex_ensure!(buffers.is_empty(), "RLEArray expects 0 buffers, got {}", buffers.len()); + vortex_ensure!( + buffers.is_empty(), + "RLEArray expects 0 buffers, got {}", + buffers.len() + ); let metadata = RLEMetadata::decode(metadata)?; let values = children.get( 0, @@ -211,7 +216,7 @@ impl RLE { ) -> VortexResult { let dtype = DType::Primitive(values.dtype().as_ptype(), indices.dtype().nullability()); let data = RLEData::try_new(values, indices, values_idx_offsets, offset, length)?; - Array::try_from_parts(ArrayNew::new(RLE, dtype, length, data)) + Array::try_from_parts(ArrayParts::new(RLE, dtype, length, data)) } /// Create a new RLE array without validation. @@ -227,7 +232,7 @@ impl RLE { ) -> RLEArray { let dtype = DType::Primitive(values.dtype().as_ptype(), indices.dtype().nullability()); let data = unsafe { RLEData::new_unchecked(values, indices, values_idx_offsets, offset) }; - Array::try_from_parts(ArrayNew::new(RLE, dtype, length, data)) + Array::try_from_parts(ArrayParts::new(RLE, dtype, length, data)) .vortex_expect("pre-validated RLE parts must be valid") } diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index a93ef931e11..62874cfc3dc 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -11,10 +11,10 @@ use fsst::Decompressor; use fsst::Symbol; use prost::Message as _; use vortex_array::Array; -use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::Canonical; @@ -132,11 +132,13 @@ impl VTable for FSST { } fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some(FSSTMetadata { - uncompressed_lengths_ptype: array.uncompressed_lengths().dtype().as_ptype().into(), - codes_offsets_ptype: array.codes.offsets().dtype().as_ptype().into(), - } - .encode_to_vec())) + Ok(Some( + FSSTMetadata { + uncompressed_lengths_ptype: array.uncompressed_lengths().dtype().as_ptype().into(), + codes_offsets_ptype: array.codes.offsets().dtype().as_ptype().into(), + } + .encode_to_vec(), + )) } fn deserialize( @@ -176,13 +178,7 @@ impl VTable for FSST { len, )?; - return FSSTData::try_new( - symbols, - symbol_lengths, - codes, - uncompressed_lengths, - dtype, - ); + return FSSTData::try_new(symbols, symbol_lengths, codes, uncompressed_lengths, dtype); } // Check for the current deserialization path. @@ -223,13 +219,7 @@ impl VTable for FSST { codes_validity, )?; - return FSSTData::try_new( - symbols, - symbol_lengths, - codes, - uncompressed_lengths, - dtype, - ); + return FSSTData::try_new(symbols, symbol_lengths, codes, uncompressed_lengths, dtype); } vortex_bail!( @@ -371,14 +361,8 @@ impl FSST { uncompressed_lengths: ArrayRef, ) -> VortexResult { let len = codes.len(); - let data = FSSTData::try_new( - symbols, - symbol_lengths, - codes, - uncompressed_lengths, - &dtype, - )?; - Array::try_from_parts(ArrayNew::new(FSST, dtype, len, data)) + let data = FSSTData::try_new(symbols, symbol_lengths, codes, uncompressed_lengths, &dtype)?; + Array::try_from_parts(ArrayParts::new(FSST, dtype, len, data)) } pub(crate) unsafe fn new_unchecked( @@ -389,8 +373,10 @@ impl FSST { uncompressed_lengths: ArrayRef, ) -> FSSTArray { let len = codes.len(); - let data = unsafe { FSSTData::new_unchecked(symbols, symbol_lengths, codes, uncompressed_lengths) }; - Array::try_from_parts(ArrayNew::new(FSST, dtype, len, data)) + let data = unsafe { + FSSTData::new_unchecked(symbols, symbol_lengths, codes, uncompressed_lengths) + }; + Array::try_from_parts(ArrayParts::new(FSST, dtype, len, data)) .vortex_expect("pre-validated FSST parts must be valid") } } @@ -411,10 +397,24 @@ impl FSSTData { uncompressed_lengths: ArrayRef, dtype: &DType, ) -> VortexResult { - Self::validate_parts(&symbols, &symbol_lengths, &codes, &uncompressed_lengths, dtype, codes.len())?; + Self::validate_parts( + &symbols, + &symbol_lengths, + &codes, + &uncompressed_lengths, + dtype, + codes.len(), + )?; // SAFETY: all components validated above - unsafe { Ok(Self::new_unchecked(symbols, symbol_lengths, codes, uncompressed_lengths)) } + unsafe { + Ok(Self::new_unchecked( + symbols, + symbol_lengths, + codes, + uncompressed_lengths, + )) + } } pub fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { diff --git a/encodings/parquet-variant/src/array.rs b/encodings/parquet-variant/src/array.rs index a0c2a47d286..21ddf65f0f3 100644 --- a/encodings/parquet-variant/src/array.rs +++ b/encodings/parquet-variant/src/array.rs @@ -8,7 +8,7 @@ use arrow_array::ArrayRef as ArrowArrayRef; use arrow_schema::Field; use parquet_variant_compute::VariantArray as ArrowVariantArray; use vortex_array::Array; -use vortex_array::ArrayNew; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; @@ -80,7 +80,7 @@ impl ParquetVariant { let len = metadata.len(); let dtype = DType::Variant(validity.nullability()); let data = ParquetVariantData::try_new(validity, metadata, value, typed_value)?; - Array::try_from_parts(ArrayNew::new(ParquetVariant, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(ParquetVariant, dtype, len, data)) } } @@ -147,14 +147,14 @@ impl ParquetVariantData { &DType::Binary(vortex_array::dtype::Nullability::NonNullable), "metadata dtype must be non-nullable binary" ); - vortex_ensure_eq!(metadata.len(), len, "metadata length must match array length"); + vortex_ensure_eq!( + metadata.len(), + len, + "metadata length must match array length" + ); if let Some(validity_len) = validity.maybe_len() { - vortex_ensure_eq!( - validity_len, - len, - "validity length must match array length" - ); + vortex_ensure_eq!(validity_len, len, "validity length must match array length"); } if let Some(v) = value { vortex_ensure!( @@ -165,11 +165,7 @@ impl ParquetVariantData { vortex_ensure_eq!(v.len(), len, "value length must match array length"); } if let Some(tv) = typed_value { - vortex_ensure_eq!( - tv.len(), - len, - "typed_value length must match array length" - ); + vortex_ensure_eq!(tv.len(), len, "typed_value length must match array length"); } Ok(()) } diff --git a/encodings/parquet-variant/src/vtable.rs b/encodings/parquet-variant/src/vtable.rs index e462afe9096..0bb166f12ba 100644 --- a/encodings/parquet-variant/src/vtable.rs +++ b/encodings/parquet-variant/src/vtable.rs @@ -198,11 +198,7 @@ impl VTable for ParquetVariant { child_idx += 1; let value = if proto.has_value { - let v = children.get( - child_idx, - &DType::Binary(proto.value_nullable.into()), - len, - )?; + let v = children.get(child_idx, &DType::Binary(proto.value_nullable.into()), len)?; child_idx += 1; Some(v) } else { @@ -267,8 +263,8 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::serde::ArrayParts; use vortex_array::serde::SerializeOptions; + use vortex_array::serde::SerializedArray; use vortex_array::session::ArraySessionExt; use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; @@ -297,7 +293,7 @@ mod tests { session.arrays().register(ParquetVariant); session.arrays().register(Variant); - let parts = ArrayParts::try_from(concat).unwrap(); + let parts = SerializedArray::try_from(concat).unwrap(); parts .decode(&dtype, len, &ReadContext::new(ctx.to_ids()), &session) .unwrap() diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index 184d061e8ed..93e19f2c9de 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -16,10 +16,10 @@ use pco::wrapped::FileCompressor; use pco::wrapped::FileDecompressor; use prost::Message; use vortex_array::Array; -use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; @@ -281,7 +281,7 @@ impl Pco { pub(crate) fn try_new(dtype: DType, data: PcoData) -> VortexResult { let len = data.len(); - Array::try_from_parts(ArrayNew::new(Pco, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Pco, dtype, len, data)) } /// Compress a primitive array using pcodec. @@ -356,7 +356,13 @@ impl PcoData { self.chunk_metas.len() ); vortex_ensure!( - self.pages.len() == self.metadata.chunks.iter().map(|chunk| chunk.pages.len()).sum(), + self.pages.len() + == self + .metadata + .chunks + .iter() + .map(|chunk| chunk.pages.len()) + .sum::(), "page count does not match metadata" ); Ok(()) diff --git a/encodings/pco/src/test.rs b/encodings/pco/src/test.rs index bfadb843fd4..647abaaa6aa 100644 --- a/encodings/pco/src/test.rs +++ b/encodings/pco/src/test.rs @@ -17,8 +17,8 @@ use vortex_array::assert_nth_scalar; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; -use vortex_array::serde::ArrayParts; use vortex_array::serde::SerializeOptions; +use vortex_array::serde::SerializedArray; use vortex_array::session::ArraySession; use vortex_array::session::ArraySessionExt; use vortex_array::validity::Validity; @@ -179,7 +179,7 @@ fn test_serde() -> VortexResult<()> { .collect::>() .freeze(); - let parts = ArrayParts::try_from(bytes)?; + let parts = SerializedArray::try_from(bytes)?; let decoded = parts.decode( &DType::Primitive(PType::I32, Nullability::NonNullable), 1_000_000, diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index 8d52c21858e..bdeef325baa 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -5,10 +5,10 @@ use std::fmt::Debug; use std::hash::Hash; use vortex_array::Array; -use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::DeserializeMetadata; @@ -104,13 +104,15 @@ impl VTable for RunEnd { } fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some(ProstMetadata(RunEndMetadata { - ends_ptype: PType::try_from(array.ends().dtype()).vortex_expect("Must be a valid PType") - as i32, - num_runs: array.ends().len() as u64, - offset: array.offset() as u64, - }) - .serialize())) + Ok(Some( + ProstMetadata(RunEndMetadata { + ends_ptype: PType::try_from(array.ends().dtype()) + .vortex_expect("Must be a valid PType") as i32, + num_runs: array.ends().len() as u64, + offset: array.offset() as u64, + }) + .serialize(), + )) } fn deserialize( @@ -214,7 +216,7 @@ impl RunEnd { ) -> RunEndArray { let data = unsafe { RunEndData::new_unchecked(ends, values, offset, length) }; let dtype = data.dtype().clone(); - Array::try_from_parts(ArrayNew::new(RunEnd, dtype, length, data)) + Array::try_from_parts(ArrayParts::new(RunEnd, dtype, length, data)) .vortex_expect("RunEndData is always valid") } @@ -223,7 +225,7 @@ impl RunEnd { let len = RunEndData::logical_len_from_ends(&ends)?; let data = RunEndData::try_new_offset_length(ends, values, 0, len)?; let dtype = data.dtype().clone(); - Array::try_from_parts(ArrayNew::new(RunEnd, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(RunEnd, dtype, len, data)) } /// Build a new [`RunEndArray`] from ends, values, offset, and length. @@ -235,7 +237,7 @@ impl RunEnd { ) -> VortexResult { let data = RunEndData::try_new_offset_length(ends, values, offset, length)?; let dtype = data.dtype().clone(); - Array::try_from_parts(ArrayNew::new(RunEnd, dtype, length, data)) + Array::try_from_parts(ArrayParts::new(RunEnd, dtype, length, data)) } /// Build a new [`RunEndArray`] from ends and values (panics on invalid input). @@ -248,7 +250,7 @@ impl RunEnd { let len = array.len(); let data = RunEndData::encode(array)?; let dtype = data.dtype().clone(); - Array::try_from_parts(ArrayNew::new(RunEnd, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(RunEnd, dtype, len, data)) } } diff --git a/encodings/runend/src/arrow.rs b/encodings/runend/src/arrow.rs index 82eb837f48f..674125f1f03 100644 --- a/encodings/runend/src/arrow.rs +++ b/encodings/runend/src/arrow.rs @@ -30,7 +30,9 @@ where .reinterpret_cast(R::Native::PTYPE.to_unsigned()); let values = ArrayRef::from_arrow(array.values().as_ref(), nullable)?; - let ends_array = ends.into_array(); + let ends_array = + PrimitiveArray::from_buffer_handle(ends.buffer_handle().clone(), ends.ptype(), ends.validity()) + .into_array(); let (ends_slice, values_slice) = if offset == 0 && len == array.run_ends().max_value() { (ends_array, values) } else { diff --git a/encodings/sequence/src/array.rs b/encodings/sequence/src/array.rs index 7d3dab3291f..6839c0a64c3 100644 --- a/encodings/sequence/src/array.rs +++ b/encodings/sequence/src/array.rs @@ -5,8 +5,8 @@ use std::hash::Hash; use num_traits::cast::FromPrimitive; use vortex_array::Array; -use vortex_array::ArrayNew; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::DeserializeMetadata; @@ -133,7 +133,10 @@ impl SequenceData { fn normalize(base: PValue, multiplier: PValue, ptype: PType) -> VortexResult<(PValue, PValue)> { match_each_integer_ptype!(ptype, |P| { - Ok((PValue::from(base.cast::

()?), PValue::from(multiplier.cast::

()?))) + Ok(( + PValue::from(base.cast::

()?), + PValue::from(multiplier.cast::

()?), + )) }) } @@ -263,8 +266,16 @@ impl VTable for Sequence { children: &dyn ArrayChildren, session: &VortexSession, ) -> VortexResult { - vortex_ensure!(buffers.is_empty(), "SequenceArray expects 0 buffers, got {}", buffers.len()); - vortex_ensure!(children.is_empty(), "SequenceArray expects 0 children, got {}", children.len()); + vortex_ensure!( + buffers.is_empty(), + "SequenceArray expects 0 buffers, got {}", + buffers.len() + ); + vortex_ensure!( + children.is_empty(), + "SequenceArray expects 0 children, got {}", + children.len() + ); let prost = as DeserializeMetadata>::deserialize(metadata)?; @@ -295,13 +306,7 @@ impl VTable for Sequence { .pvalue() .vortex_expect("sequence array multiplier should be a non-nullable primitive"); - SequenceData::try_new( - base, - multiplier, - ptype, - dtype.nullability(), - len, - ) + SequenceData::try_new(base, multiplier, ptype, dtype.nullability(), len) } fn slots(array: ArrayView<'_, Self>) -> &[Option] { @@ -409,7 +414,7 @@ impl Sequence { .vortex_expect("SequenceArray parts must be normalized to the target ptype"); let stats = Self::stats(multiplier); let data = unsafe { SequenceData::new_unchecked(base, multiplier) }; - Array::try_from_parts(ArrayNew::new(Sequence, dtype, length, data).with_stats(stats)) + Array::try_from_parts(ArrayParts::new(Sequence, dtype, length, data).with_stats(stats)) .vortex_expect("pre-validated SequenceArray parts must be valid") } @@ -424,7 +429,7 @@ impl Sequence { let dtype = DType::Primitive(ptype, nullability); let data = SequenceData::try_new(base, multiplier, ptype, nullability, length)?; let stats = Self::stats(data.multiplier()); - Array::try_from_parts(ArrayNew::new(Sequence, dtype, length, data).with_stats(stats)) + Array::try_from_parts(ArrayParts::new(Sequence, dtype, length, data).with_stats(stats)) } /// Construct a new typed [`SequenceArray`] from base/multiplier values. @@ -436,14 +441,9 @@ impl Sequence { ) -> VortexResult { let ptype = T::PTYPE; let dtype = DType::Primitive(ptype, nullability); - let data = SequenceData::try_new_typed( - base, - multiplier, - nullability, - length, - )?; + let data = SequenceData::try_new_typed(base, multiplier, nullability, length)?; let stats = Self::stats(data.multiplier()); - Array::try_from_parts(ArrayNew::new(Sequence, dtype, length, data).with_stats(stats)) + Array::try_from_parts(ArrayParts::new(Sequence, dtype, length, data).with_stats(stats)) } } diff --git a/encodings/sparse/src/canonical.rs b/encodings/sparse/src/canonical.rs index 9dbf83b4025..c0da00a14d7 100644 --- a/encodings/sparse/src/canonical.rs +++ b/encodings/sparse/src/canonical.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use itertools::Itertools; use num_traits::NumCast; use vortex_array::Array; -use vortex_array::ArrayNew; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; @@ -480,7 +480,7 @@ fn execute_sparse_decimal( let dtype = filled_array.dtype().clone(); let len = filled_array.len(); let data = filled_array.into_data().patch(patches, ctx)?; - Ok(Array::try_from_parts(ArrayNew::new(Decimal, dtype, len, data))?.into_array()) + Ok(Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data))?.into_array()) } fn execute_varbin( diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index 6d20fa2e461..a25dc92e437 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -7,10 +7,10 @@ use std::hash::Hash; use kernel::PARENT_KERNELS; use prost::Message as _; use vortex_array::Array; -use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; @@ -111,9 +111,7 @@ impl VTable for Sparse { fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { let patches = array.patches().to_metadata(array.len(), array.dtype())?; - let prost_patches = ProstPatchesMetadata { - patches, - }; + let prost_patches = ProstPatchesMetadata { patches }; // Note that we DO NOT serialize the fill value since that is stored in the buffers. Ok(Some(prost_patches.encode_to_vec())) @@ -255,21 +253,21 @@ impl Sparse { ) -> VortexResult { let dtype = fill_value.dtype().clone(); let data = SparseData::try_new(indices, values, len, fill_value)?; - Array::try_from_parts(ArrayNew::new(Sparse, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Sparse, dtype, len, data)) } pub fn try_new_from_patches(patches: Patches, fill_value: Scalar) -> VortexResult { let dtype = fill_value.dtype().clone(); let len = patches.array_len(); let data = SparseData::try_new_from_patches(patches, fill_value)?; - Array::try_from_parts(ArrayNew::new(Sparse, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Sparse, dtype, len, data)) } pub(crate) unsafe fn new_unchecked(patches: Patches, fill_value: Scalar) -> SparseArray { let dtype = fill_value.dtype().clone(); let len = patches.array_len(); let data = unsafe { SparseData::new_unchecked(patches, fill_value) }; - Array::try_from_parts(ArrayNew::new(Sparse, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Sparse, dtype, len, data)) .vortex_expect("pre-validated SparseArray parts must be valid") } diff --git a/encodings/zigzag/src/array.rs b/encodings/zigzag/src/array.rs index 099bedb0306..6b36d7e6c23 100644 --- a/encodings/zigzag/src/array.rs +++ b/encodings/zigzag/src/array.rs @@ -2,10 +2,10 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::Array; -use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::EmptyMetadata; @@ -51,8 +51,15 @@ impl VTable for ZigZag { fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { let expected_dtype = ZigZagData::dtype_from_encoded_dtype(data.encoded().dtype())?; - vortex_ensure!(dtype == &expected_dtype, "expected dtype {expected_dtype}, got {dtype}"); - vortex_ensure!(data.encoded().len() == len, "expected len {len}, got {}", data.encoded().len()); + vortex_ensure!( + dtype == &expected_dtype, + "expected dtype {expected_dtype}, got {dtype}" + ); + vortex_ensure!( + data.encoded().len() == len, + "expected len {len}, got {}", + data.encoded().len() + ); Ok(()) } @@ -165,7 +172,7 @@ impl ZigZag { let data = ZigZagData::try_new(encoded)?; let dtype = ZigZagData::dtype_from_encoded_dtype(data.encoded().dtype())?; let len = data.len(); - Array::try_from_parts(ArrayNew::new(ZigZag, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(ZigZag, dtype, len, data)) } } @@ -187,7 +194,9 @@ impl ZigZagData { Self::dtype_from_encoded_dtype(&encoded_dtype)?; - Ok(Self { slots: vec![Some(encoded)] }) + Ok(Self { + slots: vec![Some(encoded)], + }) } /// Returns the length of the array. diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index ea1d19e49b5..b0ce52a761d 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -8,10 +8,10 @@ use std::sync::Arc; use itertools::Itertools as _; use prost::Message as _; use vortex_array::Array; -use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::Canonical; @@ -265,7 +265,7 @@ impl Zstd { pub fn try_new(dtype: DType, data: ZstdData) -> VortexResult { let len = data.len(); - Array::try_from_parts(ArrayNew::new(Zstd, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Zstd, dtype, len, data)) } /// Compress a [`VarBinViewArray`] using Zstd without a dictionary. @@ -482,7 +482,10 @@ impl ZstdData { pub fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { vortex_ensure!( - matches!(dtype, DType::Primitive(..) | DType::Binary(_) | DType::Utf8(_)), + matches!( + dtype, + DType::Primitive(..) | DType::Binary(_) | DType::Utf8(_) + ), "Unsupported dtype for Zstd array: {dtype}" ); vortex_ensure!( @@ -1036,7 +1039,10 @@ impl OperationsVTable for Zstd { _ctx: &mut ExecutionCtx, ) -> VortexResult { let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let sliced = Zstd::try_new(array.dtype().clone(), array.data().with_slice(index, index + 1))?; + let sliced = Zstd::try_new( + array.dtype().clone(), + array.data().with_slice(index, index + 1), + )?; Zstd::decompress(&sliced, &mut ctx)?.scalar_at(0) } } diff --git a/encodings/zstd/src/zstd_buffers.rs b/encodings/zstd/src/zstd_buffers.rs index 0b1439e27d9..1e74a69a9a0 100644 --- a/encodings/zstd/src/zstd_buffers.rs +++ b/encodings/zstd/src/zstd_buffers.rs @@ -7,10 +7,10 @@ use std::sync::Arc; use prost::Message as _; use vortex_array::Array; -use vortex_array::ArrayNew; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; @@ -43,8 +43,12 @@ pub struct ZstdBuffers; impl ZstdBuffers { pub const ID: ArrayId = ArrayId::new_ref("vortex.zstd_buffers"); - pub fn try_new(dtype: DType, len: usize, data: ZstdBuffersData) -> VortexResult { - Array::try_from_parts(ArrayNew::new(ZstdBuffers, dtype, len, data)) + pub fn try_new( + dtype: DType, + len: usize, + data: ZstdBuffersData, + ) -> VortexResult { + Array::try_from_parts(ArrayParts::new(ZstdBuffers, dtype, len, data)) } pub fn compress(array: &ArrayRef, level: i32) -> VortexResult { @@ -87,7 +91,9 @@ impl ZstdBuffers { buffer_handles: &[BufferHandle], session: &VortexSession, ) -> VortexResult { - array.data().build_inner(array.dtype(), array.len(), buffer_handles, session) + array + .data() + .build_inner(array.dtype(), array.len(), buffer_handles, session) } fn decompress_and_build_inner( @@ -414,13 +420,15 @@ impl VTable for ZstdBuffers { } fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some(ZstdBuffersMetadata { - inner_encoding_id: array.inner_encoding_id.to_string(), - inner_metadata: array.inner_metadata.clone(), - uncompressed_sizes: array.uncompressed_sizes.clone(), - buffer_alignments: array.buffer_alignments.clone(), - } - .encode_to_vec())) + Ok(Some( + ZstdBuffersMetadata { + inner_encoding_id: array.inner_encoding_id.to_string(), + inner_metadata: array.inner_metadata.clone(), + uncompressed_sizes: array.uncompressed_sizes.clone(), + buffer_alignments: array.buffer_alignments.clone(), + } + .encode_to_vec(), + )) } fn deserialize( diff --git a/vortex-array/src/array/typed.rs b/vortex-array/src/array/typed.rs index c2f3dc74583..8f1d1b50f49 100644 --- a/vortex-array/src/array/typed.rs +++ b/vortex-array/src/array/typed.rs @@ -26,7 +26,7 @@ use crate::stats::StatsSetRef; use crate::validity::Validity; /// Construction parameters for [`ArrayInner`] / [`Array`]. -pub struct ArrayNew { +pub struct ArrayParts { pub vtable: V, pub dtype: DType, pub len: usize, @@ -34,7 +34,7 @@ pub struct ArrayNew { pub stats: ArrayStats, } -impl ArrayNew { +impl ArrayParts { pub fn new(vtable: V, dtype: DType, len: usize, data: V::ArrayData) -> Self { Self { vtable, @@ -72,9 +72,11 @@ pub(crate) struct ArrayInner { impl ArrayInner { /// Create a new inner array from explicit construction parameters. #[doc(hidden)] - pub fn try_new(new: ArrayNew) -> VortexResult { + pub fn try_new(new: ArrayParts) -> VortexResult { new.vtable.validate(&new.data, &new.dtype, new.len)?; - Ok(unsafe { Self::from_data_unchecked(new.vtable, new.dtype, new.len, new.data, new.stats) }) + Ok(unsafe { + Self::from_data_unchecked(new.vtable, new.dtype, new.len, new.data, new.stats) + }) } /// Create without validation. @@ -165,7 +167,7 @@ pub struct Array { #[allow(clippy::same_name_method)] impl Array { /// Create a typed array from explicit construction parameters. - pub fn try_from_parts(new: ArrayNew) -> VortexResult { + pub fn try_from_parts(new: ArrayParts) -> VortexResult { let inner = ArrayRef::from_inner(Arc::new(ArrayInner::::try_new(new)?)); Ok(Self { inner, @@ -177,14 +179,10 @@ impl Array { /// /// # Safety /// Caller must ensure the provided parts are logically consistent. - pub(crate) unsafe fn from_parts_unchecked(new: ArrayNew) -> Self { + pub(crate) unsafe fn from_parts_unchecked(new: ArrayParts) -> Self { let inner = ArrayRef::from_inner(Arc::new(unsafe { ArrayInner::::from_data_unchecked( - new.vtable, - new.dtype, - new.len, - new.data, - new.stats, + new.vtable, new.dtype, new.len, new.data, new.stats, ) })); Self { diff --git a/vortex-array/src/array/vtable/dyn_.rs b/vortex-array/src/array/vtable/dyn_.rs index 458870c825a..8afb9f80f9c 100644 --- a/vortex-array/src/array/vtable/dyn_.rs +++ b/vortex-array/src/array/vtable/dyn_.rs @@ -15,8 +15,8 @@ use crate::ExecutionResult; use crate::ExecutionStep; use crate::IntoArray; use crate::array::Array; -use crate::array::ArrayNew; use crate::array::ArrayId; +use crate::array::ArrayParts; use crate::array::VTable; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -87,7 +87,7 @@ impl DynVTable for V { ) -> VortexResult { let inner = self.deserialize(dtype, len, metadata, buffers, children, session)?; Ok(Array::::try_from_parts( - ArrayNew::new(self.clone(), dtype.clone(), len, inner) + ArrayParts::new(self.clone(), dtype.clone(), len, inner) .with_stats(ArrayStats::default()), )? .into_array()) @@ -101,8 +101,8 @@ impl DynVTable for V { V::with_slots(&mut data, slots)?; Ok(unsafe { Array::::from_parts_unchecked( - ArrayNew::new(self.clone(), array.dtype().clone(), array.len(), data) - .with_stats(array.statistics().to_array_stats()), + ArrayParts::new(self.clone(), array.dtype().clone(), array.len(), data) + .with_stats(array.statistics().to_array_stats()), ) .into_array() }) diff --git a/vortex-array/src/arrays/bool/array.rs b/vortex-array/src/arrays/bool/array.rs index 56b716e9fcf..3f32ecbae50 100644 --- a/vortex-array/src/arrays/bool/array.rs +++ b/vortex-array/src/arrays/bool/array.rs @@ -12,14 +12,14 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Bool; use crate::arrays::BoolArray; use crate::buffer::BufferHandle; use crate::dtype::DType; -use crate::stats::ArrayStats; +use crate::dtype::Nullability; use crate::validity::Validity; /// The validity bitmap indicating which elements are non-null. @@ -63,11 +63,10 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; pub struct BoolData { /// Child arrays stored as slots. See [`VTable::slots`] for design rationale. pub(super) slots: Vec>, - pub(super) dtype: DType, + pub(super) nullability: Nullability, pub(super) bits: BufferHandle, pub(super) offset: usize, pub(super) len: usize, - pub(super) stats_set: ArrayStats, } pub struct BoolArrayParts { @@ -85,8 +84,8 @@ impl BoolData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + DType::Bool(self.nullability) } /// Returns `true` if this array is empty. @@ -97,7 +96,7 @@ impl BoolData { /// Returns the [`Validity`] of this array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) } /// Returns the validity as a [`Mask`]. @@ -187,10 +186,10 @@ impl Array { /// /// Returns an error if the provided components do not satisfy the invariants. pub fn try_new(bits: BitBuffer, validity: Validity) -> VortexResult { + let dtype = DType::Bool(validity.nullability()); + let len = bits.len(); let data = BoolData::try_new(bits, validity)?; - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(Bool, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Bool, dtype, len, data)) } /// Build a new bool array from a `BufferHandle`, returning an error if the offset is @@ -201,10 +200,9 @@ impl Array { len: usize, validity: Validity, ) -> VortexResult { + let dtype = DType::Bool(validity.nullability()); let data = BoolData::try_new_from_handle(bits, offset, len, validity)?; - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(Bool, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Bool, dtype, len, data)) } /// Creates a new [`BoolArray`] without validation. @@ -213,11 +211,11 @@ impl Array { /// /// The caller must ensure that the validity length is equal to the bit buffer length. pub unsafe fn new_unchecked(bits: BitBuffer, validity: Validity) -> Self { + let dtype = DType::Bool(validity.nullability()); + let len = bits.len(); // SAFETY: caller guarantees validity length equals bit buffer length. let data = unsafe { BoolData::new_unchecked(bits, validity) }; - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(Bool, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Bool, dtype, len, data)) .vortex_expect("BoolData is always valid") } @@ -260,15 +258,12 @@ impl BoolData { let (offset, len, buffer) = bits.into_inner(); let slots = Self::make_slots(&validity, len); - let dtype = DType::Bool(validity.nullability()); - Ok(Self { slots, - dtype, + nullability: validity.nullability(), bits: BufferHandle::new_host(buffer), offset, len, - stats_set: ArrayStats::default(), }) } @@ -294,15 +289,12 @@ impl BoolData { ); let slots = Self::make_slots(&validity, len); - let dtype = DType::Bool(validity.nullability()); - Ok(Self { slots, - dtype, + nullability: validity.nullability(), bits, offset, len, - stats_set: ArrayStats::default(), }) } @@ -312,15 +304,13 @@ impl BoolData { } else { let (offset, len, buffer) = bits.into_inner(); let slots = Self::make_slots(&validity, len); - let dtype = DType::Bool(validity.nullability()); Self { slots, - dtype, + nullability: validity.nullability(), bits: BufferHandle::new_host(buffer), offset, len, - stats_set: ArrayStats::default(), } } } diff --git a/vortex-array/src/arrays/bool/vtable/mod.rs b/vortex-array/src/arrays/bool/vtable/mod.rs index a2608f09011..5c192cdec50 100644 --- a/vortex-array/src/arrays/bool/vtable/mod.rs +++ b/vortex-array/src/arrays/bool/vtable/mod.rs @@ -36,7 +36,6 @@ use crate::array::ArrayId; use crate::arrays::bool::compute::rules::RULES; use crate::hash::ArrayEq; use crate::hash::ArrayHash; -use crate::stats::ArrayStats; vtable!(Bool, Bool, BoolData); @@ -89,16 +88,38 @@ impl VTable for Bool { fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { assert!(array.offset < 8, "Offset must be <8, got {}", array.offset); - Ok(Some(ProstMetadata(BoolMetadata { - offset: u32::try_from(array.offset).vortex_expect("checked"), - }) - .serialize())) + Ok(Some( + ProstMetadata(BoolMetadata { + offset: u32::try_from(array.offset).vortex_expect("checked"), + }) + .serialize(), + )) + } + + fn validate(&self, data: &BoolData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.len() == len, + "BoolArray length {} does not match outer length {}", + data.len(), + len + ); + + let actual_dtype = data.dtype(); + vortex_ensure!( + &actual_dtype == dtype, + "BoolArray dtype {} does not match outer dtype {}", + actual_dtype, + dtype + ); + + Ok(()) } fn deserialize( &self, dtype: &DType, - len: usize, metadata: &[u8], + len: usize, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, @@ -181,8 +202,8 @@ mod tests { use crate::LEGACY_SESSION; use crate::arrays::BoolArray; use crate::assert_arrays_eq; - use crate::serde::ArrayParts; use crate::serde::SerializeOptions; + use crate::serde::SerializedArray; #[test] fn test_nullable_bool_serde_roundtrip() { @@ -201,7 +222,7 @@ mod tests { for buf in serialized { concat.extend_from_slice(buf.as_ref()); } - let parts = ArrayParts::try_from(concat.freeze()).unwrap(); + let parts = SerializedArray::try_from(concat.freeze()).unwrap(); let decoded = parts .decode( &dtype, diff --git a/vortex-array/src/arrays/chunked/array.rs b/vortex-array/src/arrays/chunked/array.rs index 80661fb03bd..0d865e8846d 100644 --- a/vortex-array/src/arrays/chunked/array.rs +++ b/vortex-array/src/arrays/chunked/array.rs @@ -17,7 +17,7 @@ use vortex_error::vortex_bail; use crate::ArrayRef; use crate::IntoArray; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::arrays::Chunked; use crate::arrays::primitive::PrimitiveData; use crate::dtype::DType; @@ -25,7 +25,6 @@ use crate::iter::ArrayIterator; use crate::iter::ArrayIteratorAdapter; use crate::search_sorted::SearchSorted; use crate::search_sorted::SearchSortedSide; -use crate::stats::ArrayStats; use crate::stream::ArrayStream; use crate::stream::ArrayStreamAdapter; use crate::validity::Validity; @@ -36,11 +35,9 @@ pub(super) const CHUNKS_OFFSET: usize = 1; #[derive(Clone, Debug)] pub struct ChunkedData { pub(super) dtype: DType, - pub(super) len: usize, pub(super) chunk_offsets: PrimitiveData, pub(super) chunks: Vec, pub(super) slots: Vec>, - pub(super) stats_set: ArrayStats, } impl ChunkedData { @@ -51,7 +48,7 @@ impl ChunkedData { ) -> Vec> { let mut slots = Vec::with_capacity(1 + chunks.len()); slots.push(Some( - Array::try_from_parts(ArrayNew::new( + Array::try_from_parts(ArrayParts::new( crate::arrays::Primitive, chunk_offsets.dtype().clone(), chunk_offsets.len(), @@ -109,13 +106,9 @@ impl ChunkedData { let slots = Self::make_slots(&chunk_offsets, &chunks); Self { dtype, - len: curr_offset - .try_into() - .vortex_expect("chunk offset must fit in usize"), chunk_offsets, chunks, slots, - stats_set: Default::default(), } } @@ -134,7 +127,11 @@ impl ChunkedData { /// Returns the length of this array. pub fn len(&self) -> usize { - self.len + self.chunk_offsets() + .last() + .copied() + .and_then(|len| usize::try_from(len).ok()) + .vortex_expect("chunk offset must fit in usize") } /// Returns the [`DType`] of this array. @@ -229,10 +226,12 @@ impl ChunkedData { new_chunks.push( // SAFETY: chunks_to_combine contains valid chunks of the same dtype as self. // All chunks are guaranteed to be valid arrays matching self.dtype(). - unsafe { Array::::new_unchecked(chunks_to_combine, self.dtype().clone()) } - .into_array() - .to_canonical()? - .into_array(), + unsafe { + Array::::new_unchecked(chunks_to_combine, self.dtype().clone()) + } + .into_array() + .to_canonical()? + .into_array(), ); new_chunk_n_bytes = 0; @@ -272,7 +271,7 @@ impl Array { let data = ChunkedData::try_new(chunks, dtype)?; let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(Chunked, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Chunked, dtype, len, data)) } /// Creates a new `ChunkedArray` without validation. @@ -284,7 +283,7 @@ impl Array { let data = unsafe { ChunkedData::new_unchecked(chunks, dtype) }; let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(Chunked, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Chunked, dtype, len, data)) .vortex_expect("ChunkedData is always valid") } } diff --git a/vortex-array/src/arrays/chunked/vtable/mod.rs b/vortex-array/src/arrays/chunked/vtable/mod.rs index 6038296387c..30e5dd49232 100644 --- a/vortex-array/src/arrays/chunked/vtable/mod.rs +++ b/vortex-array/src/arrays/chunked/vtable/mod.rs @@ -37,7 +37,6 @@ use crate::dtype::PType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; mod canonical; @@ -105,6 +104,22 @@ impl VTable for Chunked { Ok(Some(vec![])) } + fn validate(&self, data: &ChunkedData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.len() == len, + "ChunkedArray length {} does not match outer length {}", + data.len(), + len + ); + vortex_ensure!( + data.dtype() == dtype, + "ChunkedArray dtype {} does not match outer dtype {}", + data.dtype(), + dtype + ); + Ok(()) + } + fn deserialize( &self, dtype: &DType, @@ -147,21 +162,13 @@ impl VTable for Chunked { let chunk_offsets = PrimitiveData::new(chunk_offsets_buf.clone(), Validity::NonNullable); - let total_len = chunk_offsets_buf - .last() - .ok_or_else(|| vortex_err!("chunk_offsets must not be empty"))?; - let len = usize::try_from(*total_len) - .map_err(|_| vortex_err!("total length {} exceeds usize range", total_len))?; - let slots = ChunkedData::make_slots(&chunk_offsets, &chunks); // Construct directly using the struct fields to avoid recomputing chunk_offsets Ok(ChunkedData { dtype: dtype.clone(), - len, chunk_offsets, chunks, slots, - stats_set: Default::default(), }) } @@ -204,12 +211,6 @@ impl VTable for Chunked { array.chunks = chunks; array.slots = slots; - let total_len = chunk_offsets_buf - .last() - .ok_or_else(|| vortex_err!("chunk_offsets must not be empty"))?; - array.len = usize::try_from(*total_len) - .map_err(|_| vortex_err!("total length {} exceeds usize range", total_len))?; - Ok(()) } diff --git a/vortex-array/src/arrays/constant/array.rs b/vortex-array/src/arrays/constant/array.rs index 4475e9dbafd..e4b3bb7b20e 100644 --- a/vortex-array/src/arrays/constant/array.rs +++ b/vortex-array/src/arrays/constant/array.rs @@ -5,49 +5,25 @@ use vortex_error::VortexExpect; use crate::ArrayRef; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::arrays::Constant; -use crate::dtype::DType; use crate::scalar::Scalar; -use crate::stats::ArrayStats; pub(super) const NUM_SLOTS: usize = 0; #[derive(Clone, Debug)] pub struct ConstantData { pub(super) scalar: Scalar, - pub(super) len: usize, pub(super) slots: Vec>, - pub(super) stats_set: ArrayStats, } impl ConstantData { - pub fn new(scalar: S, len: usize) -> Self + pub fn new(scalar: S) -> Self where S: Into, { let scalar = scalar.into(); - Self { - scalar, - len, - slots: vec![], - stats_set: Default::default(), - } - } - - /// Returns the length of this array. - pub fn len(&self) -> usize { - self.len - } - - /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - self.scalar.dtype() - } - - /// Returns `true` if this array is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 + Self { scalar, slots: vec![] } } /// Returns the [`Scalar`] value of this constant array. @@ -67,8 +43,8 @@ impl Array { { let scalar = scalar.into(); let dtype = scalar.dtype().clone(); - let data = ConstantData::new(scalar, len); - Array::try_from_parts(ArrayNew::new(Constant, dtype, len, data)) + let data = ConstantData::new(scalar); + Array::try_from_parts(ArrayParts::new(Constant, dtype, len, data)) .vortex_expect("ConstantData is always valid") } } diff --git a/vortex-array/src/arrays/constant/vtable/mod.rs b/vortex-array/src/arrays/constant/vtable/mod.rs index b714ba0a30a..55869cbd220 100644 --- a/vortex-array/src/arrays/constant/vtable/mod.rs +++ b/vortex-array/src/arrays/constant/vtable/mod.rs @@ -39,7 +39,6 @@ use crate::scalar::DecimalValue; use crate::scalar::Scalar; use crate::scalar::ScalarValue; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::vtable; pub(crate) mod canonical; mod operations; @@ -64,6 +63,14 @@ impl VTable for Constant { Self::ID } + fn validate(&self, data: &ConstantData, dtype: &DType, _len: usize) -> VortexResult<()> { + vortex_ensure!( + data.scalar.dtype() == dtype, + "ConstantArray scalar dtype does not match outer dtype" + ); + Ok(()) + } + fn array_hash( array: &ConstantData, state: &mut H, @@ -123,7 +130,8 @@ impl VTable for Constant { fn deserialize( &self, dtype: &DType, - len: usize, _metadata: &[u8], + _len: usize, + _metadata: &[u8], buffers: &[BufferHandle], _children: &dyn ArrayChildren, @@ -141,7 +149,7 @@ impl VTable for Constant { let scalar_value = ScalarValue::from_proto_bytes(bytes, dtype, session)?; let scalar = Scalar::try_new(dtype.clone(), scalar_value)?; - Ok(ConstantData::new(scalar, len)) + Ok(ConstantData::new(scalar)) } fn reduce_parent( diff --git a/vortex-array/src/arrays/decimal/array.rs b/vortex-array/src/arrays/decimal/array.rs index 35124837d9a..cc6b234846d 100644 --- a/vortex-array/src/arrays/decimal/array.rs +++ b/vortex-array/src/arrays/decimal/array.rs @@ -15,7 +15,7 @@ use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Decimal; @@ -28,10 +28,10 @@ use crate::dtype::DecimalDType; use crate::dtype::DecimalType; use crate::dtype::IntegerPType; use crate::dtype::NativeDecimalType; +use crate::dtype::Nullability; use crate::match_each_decimal_value_type; use crate::match_each_integer_ptype; use crate::patches::Patches; -use crate::stats::ArrayStats; use crate::validity::Validity; /// The validity bitmap indicating which elements are non-null. @@ -99,10 +99,10 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; #[derive(Clone, Debug)] pub struct DecimalData { pub(super) slots: Vec>, - pub(super) dtype: DType, + pub(super) decimal_dtype: DecimalDType, + pub(super) nullability: Nullability, pub(super) values: BufferHandle, pub(super) values_type: DecimalType, - pub(super) stats_set: ArrayStats, } pub struct DecimalArrayParts { @@ -243,8 +243,8 @@ impl DecimalData { slots: Self::make_slots(&validity, len), values, values_type, - dtype: DType::Decimal(decimal_dtype, validity.nullability()), - stats_set: Default::default(), + decimal_dtype, + nullability: validity.nullability(), } } @@ -301,8 +301,8 @@ impl DecimalData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + DType::Decimal(self.decimal_dtype, self.nullability) } /// Returns `true` if this array is empty. @@ -312,15 +312,13 @@ impl DecimalData { /// Reconstructs the validity from the slot state. pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) } pub fn into_parts(self) -> DecimalArrayParts { let validity = self.validity(); - let decimal_dtype = self.dtype.into_decimal_opt().vortex_expect("cannot fail"); - DecimalArrayParts { - decimal_dtype, + decimal_dtype: self.decimal_dtype, values: self.values, values_type: self.values_type, validity, @@ -345,11 +343,7 @@ impl DecimalData { /// Returns the decimal type information pub fn decimal_dtype(&self) -> DecimalDType { - if let DType::Decimal(decimal_dtype, _) = self.dtype { - decimal_dtype - } else { - vortex_panic!("Expected Decimal dtype, got {:?}", self.dtype) - } + self.decimal_dtype } /// Return the `DecimalType` used to represent the values in the array. @@ -451,10 +445,10 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> Self { + let dtype = DType::Decimal(decimal_dtype, validity.nullability()); + let len = buffer.len(); let data = DecimalData::new(buffer, decimal_dtype, validity); - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(Decimal, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data)) .vortex_expect("DecimalData is always valid") } @@ -468,10 +462,10 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> Self { + let dtype = DType::Decimal(decimal_dtype, validity.nullability()); + let len = buffer.len(); let data = unsafe { DecimalData::new_unchecked(buffer, decimal_dtype, validity) }; - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(Decimal, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data)) .vortex_expect("DecimalData is always valid") } @@ -481,10 +475,10 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> VortexResult { + let dtype = DType::Decimal(decimal_dtype, validity.nullability()); + let len = buffer.len(); let data = DecimalData::try_new(buffer, decimal_dtype, validity)?; - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(Decimal, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data)) } /// Creates a new [`DecimalArray`] from an iterator of values. @@ -497,9 +491,9 @@ impl Array { decimal_dtype: DecimalDType, ) -> Self { let data = DecimalData::from_iter(iter, decimal_dtype); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(Decimal, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data)) .vortex_expect("DecimalData is always valid") } @@ -509,9 +503,9 @@ impl Array { decimal_dtype: DecimalDType, ) -> Self { let data = DecimalData::from_option_iter(iter, decimal_dtype); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(Decimal, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data)) .vortex_expect("DecimalData is always valid") } @@ -522,11 +516,11 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> Self { + let dtype = DType::Decimal(decimal_dtype, validity.nullability()); + let len = values.len() / values_type.byte_width(); let data = DecimalData::new_handle(values, values_type, decimal_dtype, validity); - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(Decimal, dtype, len, data)) - .vortex_expect("DecimalData is always valid") + Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data)) + .vortex_expect("DecimalData is always valid") } /// Creates a new [`DecimalArray`] without validation from a [`BufferHandle`]. @@ -540,13 +534,13 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> Self { + let dtype = DType::Decimal(decimal_dtype, validity.nullability()); + let len = values.len() / values_type.byte_width(); let data = unsafe { DecimalData::new_unchecked_handle(values, values_type, decimal_dtype, validity) }; - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(Decimal, dtype, len, data)) - .vortex_expect("DecimalData is always valid") + Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data)) + .vortex_expect("DecimalData is always valid") } } diff --git a/vortex-array/src/arrays/decimal/compute/between.rs b/vortex-array/src/arrays/decimal/compute/between.rs index fd8627c898c..b521d2148e0 100644 --- a/vortex-array/src/arrays/decimal/compute/between.rs +++ b/vortex-array/src/arrays/decimal/compute/between.rs @@ -36,7 +36,7 @@ impl BetweenKernel for Decimal { // NOTE: we know that have checked before that the lower and upper bounds are not all null. let nullability = - arr.dtype.nullability() | lower.dtype().nullability() | upper.dtype().nullability(); + arr.dtype().nullability() | lower.dtype().nullability() | upper.dtype().nullability(); match_each_decimal_value_type!(arr.values_type(), |D| { between_unpack::(arr, lower, upper, nullability, options) diff --git a/vortex-array/src/arrays/decimal/vtable/mod.rs b/vortex-array/src/arrays/decimal/vtable/mod.rs index 8c2b8a9e65c..bd24cd313ba 100644 --- a/vortex-array/src/arrays/decimal/vtable/mod.rs +++ b/vortex-array/src/arrays/decimal/vtable/mod.rs @@ -40,7 +40,6 @@ use crate::arrays::decimal::array::SLOT_NAMES; use crate::arrays::decimal::compute::rules::RULES; use crate::hash::ArrayEq; use crate::hash::ArrayHash; -use crate::stats::ArrayStats; vtable!(Decimal, Decimal, DecimalData); // The type of the values can be determined by looking at the type info...right? @@ -91,16 +90,38 @@ impl VTable for Decimal { } fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some(ProstMetadata(DecimalMetadata { - values_type: array.values_type() as i32, - }) - .serialize())) + Ok(Some( + ProstMetadata(DecimalMetadata { + values_type: array.values_type() as i32, + }) + .serialize(), + )) + } + + fn validate(&self, data: &DecimalData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.len() == len, + "DecimalArray length {} does not match outer length {}", + data.len(), + len + ); + + let actual_dtype = data.dtype(); + vortex_ensure!( + &actual_dtype == dtype, + "DecimalArray dtype {} does not match outer dtype {}", + actual_dtype, + dtype + ); + + Ok(()) } fn deserialize( &self, dtype: &DType, - len: usize, metadata: &[u8], + len: usize, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, @@ -197,8 +218,8 @@ mod tests { use crate::arrays::DecimalArray; use crate::assert_arrays_eq; use crate::dtype::DecimalDType; - use crate::serde::ArrayParts; use crate::serde::SerializeOptions; + use crate::serde::SerializedArray; use crate::validity::Validity; #[test] @@ -223,7 +244,7 @@ mod tests { let concat = concat.freeze(); - let parts = ArrayParts::try_from(concat).unwrap(); + let parts = SerializedArray::try_from(concat).unwrap(); let decoded = parts .decode(&dtype, 5, &ReadContext::new(ctx.to_ids()), &LEGACY_SESSION) .unwrap(); @@ -251,7 +272,7 @@ mod tests { concat.extend_from_slice(buf.as_ref()); } - let parts = ArrayParts::try_from(concat.freeze()).unwrap(); + let parts = SerializedArray::try_from(concat.freeze()).unwrap(); let decoded = parts .decode( &dtype, diff --git a/vortex-array/src/arrays/dict/array.rs b/vortex-array/src/arrays/dict/array.rs index caad6c60b30..26192b6c00c 100644 --- a/vortex-array/src/arrays/dict/array.rs +++ b/vortex-array/src/arrays/dict/array.rs @@ -11,12 +11,11 @@ use vortex_mask::AllOr; use crate::ArrayRef; use crate::ToCanonical; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::arrays::Dict; use crate::dtype::DType; use crate::dtype::PType; use crate::match_each_integer_ptype; -use crate::stats::ArrayStats; #[derive(Clone, prost::Message)] pub struct DictMetadata { @@ -44,8 +43,6 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["codes", "values"]; #[derive(Debug, Clone)] pub struct DictData { pub(super) slots: Vec>, - pub(super) stats_set: ArrayStats, - pub(super) dtype: DType, /// Indicates whether all dictionary values are definitely referenced by at least one code. /// `true` = all values are referenced (computed during encoding). /// `false` = unknown/might have unreferenced values. @@ -57,7 +54,6 @@ pub struct DictData { pub struct DictArrayParts { pub codes: ArrayRef, pub values: ArrayRef, - pub dtype: DType, } impl DictData { @@ -68,13 +64,8 @@ impl DictData { /// by the safe `DictArray::try_new` constructor are valid, for example when /// you are filtering or slicing an existing valid `DictArray`. pub unsafe fn new_unchecked(codes: ArrayRef, values: ArrayRef) -> Self { - let dtype = values - .dtype() - .union_nullability(codes.dtype().nullability()); Self { slots: vec![Some(codes), Some(values)], - stats_set: Default::default(), - dtype, all_values_referenced: false, } } @@ -134,8 +125,10 @@ impl DictData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + self.values() + .dtype() + .union_nullability(self.codes().dtype().nullability()) } /// Returns `true` if this array is empty. @@ -151,7 +144,6 @@ impl DictData { values: self.slots[VALUES_SLOT] .take() .vortex_expect("DictArray values slot"), - dtype: self.dtype, } } @@ -206,18 +198,18 @@ impl Array { /// Build a new `DictArray` from its components, `codes` and `values`. pub fn new(codes: ArrayRef, values: ArrayRef) -> Self { let data = DictData::new(codes, values); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(Dict, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Dict, dtype, len, data)) .vortex_expect("DictData is always valid") } /// Build a new `DictArray` from its components, `codes` and `values`. pub fn try_new(codes: ArrayRef, values: ArrayRef) -> VortexResult { let data = DictData::try_new(codes, values)?; - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(Dict, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Dict, dtype, len, data)) } /// Build a new `DictArray` without validating the codes or values. @@ -227,9 +219,9 @@ impl Array { /// See [`DictData::new_unchecked`]. pub unsafe fn new_unchecked(codes: ArrayRef, values: ArrayRef) -> Self { let data = unsafe { DictData::new_unchecked(codes, values) }; - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(Dict, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Dict, dtype, len, data)) .vortex_expect("DictData is always valid") } @@ -245,8 +237,8 @@ impl Array { self.into_data() .set_all_values_referenced(all_values_referenced) }; - Array::try_from_parts(ArrayNew::new(Dict, dtype, len, data)) - .vortex_expect("data is always valid") + Array::try_from_parts(ArrayParts::new(Dict, dtype, len, data)) + .vortex_expect("data is always valid") } } diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index bf5a1f622a0..fc61e0311f2 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -41,7 +41,6 @@ use crate::hash::ArrayHash; use crate::require_child; use crate::scalar::Scalar; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::vtable; mod kernel; mod operations; @@ -66,6 +65,15 @@ impl VTable for Dict { Self::ID } + fn validate(&self, data: &DictData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!(data.codes().len() == len, "DictArray codes length mismatch"); + vortex_ensure!( + data.dtype() == *dtype, + "DictArray dtype does not match codes/values dtype" + ); + Ok(()) + } + fn array_hash(array: &DictData, state: &mut H, precision: Precision) { array.codes().array_hash(state, precision); array.values().array_hash(state, precision); @@ -106,7 +114,8 @@ impl VTable for Dict { fn deserialize( &self, dtype: &DType, - len: usize, metadata: &[u8], + len: usize, + metadata: &[u8], _buffers: &[BufferHandle], children: &dyn ArrayChildren, diff --git a/vortex-array/src/arrays/dict/vtable/validity.rs b/vortex-array/src/arrays/dict/vtable/validity.rs index 708d05c4ce3..e9c51c4b53e 100644 --- a/vortex-array/src/arrays/dict/vtable/validity.rs +++ b/vortex-array/src/arrays/dict/vtable/validity.rs @@ -8,7 +8,6 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::array::ValidityVTable; use crate::arrays::DictArray; -use crate::arrays::dict::DictData; use crate::builtins::ArrayBuiltins; use crate::dtype::Nullability; use crate::scalar::Scalar; diff --git a/vortex-array/src/arrays/extension/array.rs b/vortex-array/src/arrays/extension/array.rs index 146ab9540b3..d20d53e0e56 100644 --- a/vortex-array/src/arrays/extension/array.rs +++ b/vortex-array/src/arrays/extension/array.rs @@ -6,11 +6,10 @@ use vortex_error::VortexResult; use crate::ArrayRef; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::arrays::Extension; use crate::dtype::DType; use crate::dtype::extension::ExtDTypeRef; -use crate::stats::ArrayStats; /// The backing storage array for this extension array. pub(super) const STORAGE_SLOT: usize = 0; @@ -58,9 +57,8 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["storage"]; #[derive(Clone, Debug)] pub struct ExtensionData { /// The storage dtype. This **must** be a [`Extension::DType`] variant. - pub(super) dtype: DType, + pub(super) ext_dtype: ExtDTypeRef, pub(super) slots: Vec>, - pub(super) stats_set: ArrayStats, } impl ExtensionData { @@ -111,9 +109,8 @@ impl ExtensionData { ); Self { - dtype: DType::Extension(ext_dtype), + ext_dtype, slots: vec![Some(storage_array)], - stats_set: ArrayStats::default(), } } @@ -123,8 +120,8 @@ impl ExtensionData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + DType::Extension(self.ext_dtype.clone()) } /// Returns `true` if this array is empty. @@ -134,11 +131,7 @@ impl ExtensionData { /// The extension dtype of this array. pub fn ext_dtype(&self) -> &ExtDTypeRef { - let DType::Extension(ext) = &self.dtype else { - unreachable!("ExtensionArray: dtype must be an ExtDType") - }; - - ext + &self.ext_dtype } pub fn storage_array(&self) -> &ArrayRef { @@ -158,7 +151,7 @@ impl Array { let dtype = DType::Extension(ext_dtype.clone()); let len = storage_array.len(); let data = ExtensionData::new(ext_dtype, storage_array); - Array::try_from_parts(ArrayNew::new(Extension, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Extension, dtype, len, data)) .vortex_expect("ExtensionData is always valid") } @@ -167,6 +160,6 @@ impl Array { let dtype = DType::Extension(ext_dtype.clone()); let len = storage_array.len(); let data = ExtensionData::try_new(ext_dtype, storage_array)?; - Array::try_from_parts(ArrayNew::new(Extension, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Extension, dtype, len, data)) } } diff --git a/vortex-array/src/arrays/extension/vtable/mod.rs b/vortex-array/src/arrays/extension/vtable/mod.rs index e61d40fd2c9..4335b88a491 100644 --- a/vortex-array/src/arrays/extension/vtable/mod.rs +++ b/vortex-array/src/arrays/extension/vtable/mod.rs @@ -31,7 +31,6 @@ use crate::dtype::DType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::vtable; vtable!(Extension, Extension, ExtensionData); @@ -84,6 +83,25 @@ impl VTable for Extension { Ok(Some(vec![])) } + fn validate(&self, data: &ExtensionData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.len() == len, + "ExtensionArray length {} does not match outer length {}", + data.len(), + len + ); + + let actual_dtype = data.dtype(); + vortex_ensure!( + &actual_dtype == dtype, + "ExtensionArray dtype {} does not match outer dtype {}", + actual_dtype, + dtype + ); + + Ok(()) + } + fn deserialize( &self, dtype: &DType, diff --git a/vortex-array/src/arrays/filter/array.rs b/vortex-array/src/arrays/filter/array.rs index ecae1086948..e4835188d2b 100644 --- a/vortex-array/src/arrays/filter/array.rs +++ b/vortex-array/src/arrays/filter/array.rs @@ -8,10 +8,9 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::arrays::Filter; use crate::dtype::DType; -use crate::stats::ArrayStats; /// The source array being filtered. pub(super) const CHILD_SLOT: usize = 0; @@ -37,9 +36,6 @@ pub struct FilterData { /// The boolean mask selecting which elements to keep. pub(super) mask: Mask, - - /// The stats for this array. - pub(super) stats: ArrayStats, } impl FilterData { @@ -59,7 +55,6 @@ impl FilterData { Ok(Self { slots: vec![Some(array)], mask, - stats: ArrayStats::default(), }) } @@ -97,7 +92,7 @@ impl Array { let dtype = array.dtype().clone(); let len = mask.true_count(); let data = FilterData::new(array, mask); - Array::try_from_parts(ArrayNew::new(Filter, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Filter, dtype, len, data)) .vortex_expect("FilterData is always valid") } @@ -106,7 +101,7 @@ impl Array { let dtype = array.dtype().clone(); let len = mask.true_count(); let data = FilterData::try_new(array, mask)?; - Array::try_from_parts(ArrayNew::new(Filter, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Filter, dtype, len, data)) } } diff --git a/vortex-array/src/arrays/filter/vtable.rs b/vortex-array/src/arrays/filter/vtable.rs index e174c298f58..90deb1790e4 100644 --- a/vortex-array/src/arrays/filter/vtable.rs +++ b/vortex-array/src/arrays/filter/vtable.rs @@ -1,8 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::fmt::Debug; -use std::fmt::Formatter; use std::hash::Hasher; use vortex_error::VortexResult; @@ -36,7 +34,6 @@ use crate::executor::ExecutionCtx; use crate::executor::ExecutionResult; use crate::scalar::Scalar; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; @@ -95,11 +92,12 @@ impl VTable for Filter { fn deserialize( &self, - dtype: &DType, - len: usize, _metadata: &[u8], + _dtype: &DType, + _len: usize, + _metadata: &[u8], _buffers: &[BufferHandle], - children: &dyn ArrayChildren, + _children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { vortex_bail!("Filter array is not serializable") @@ -159,17 +157,3 @@ impl ValidityVTable for Filter { array.child().validity()?.filter(&array.mask) } } - -pub struct FilterMetadata(pub(super) Mask); - -impl Debug for FilterMetadata { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{} / {} => {}", - self.0.true_count(), - self.0.len(), - self.0.density() - ) - } -} diff --git a/vortex-array/src/arrays/fixed_size_list/array.rs b/vortex-array/src/arrays/fixed_size_list/array.rs index 04618e28791..f5bdec71787 100644 --- a/vortex-array/src/arrays/fixed_size_list/array.rs +++ b/vortex-array/src/arrays/fixed_size_list/array.rs @@ -9,12 +9,12 @@ use vortex_error::vortex_ensure; use crate::ArrayRef; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::FixedSizeList; use crate::dtype::DType; -use crate::stats::ArrayStats; +use crate::dtype::Nullability; use crate::validity::Validity; /// The `elements` data array, where each fixed-size list scalar is a _slice_ of the `elements` @@ -81,10 +81,8 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["elements", "validity"]; /// ``` #[derive(Clone, Debug)] pub struct FixedSizeListData { - /// The [`DType`] of the fixed-size list. - /// - /// This type **must** be the variant [`DType::FixedSizeList`]. - pub(super) dtype: DType, + /// The nullability of the fixed-size list array. + pub(super) nullability: Nullability, /// Slots holding [elements]. pub(super) slots: Vec>, @@ -101,10 +99,8 @@ pub struct FixedSizeListData { /// The main reason we need to store this (rather than calculate it on the fly via `list_size` /// and `elements.len()`) is because in the degenerate case where `list_size == 0`, we cannot /// use `0 / 0` to determine the length. - pub(super) len: usize, + pub(super) degenerate_len: usize, - /// The stats for this array. - pub(super) stats_set: ArrayStats, } impl FixedSizeListData { @@ -169,11 +165,10 @@ impl FixedSizeListData { let validity_slot = validity_to_child(&validity, len); Self { - dtype: DType::FixedSizeList(Arc::new(elements.dtype().clone()), list_size, nullability), + nullability, slots: vec![Some(elements), validity_slot], list_size, - len, - stats_set: Default::default(), + degenerate_len: if list_size == 0 { len } else { 0 }, } } @@ -184,7 +179,7 @@ impl FixedSizeListData { .take() .vortex_expect("FixedSizeListArray elements slot"), validity, - self.dtype, + self.dtype(), ) } @@ -225,13 +220,21 @@ impl FixedSizeListData { } /// Returns the dtype of the array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + DType::FixedSizeList( + Arc::new(self.elements().dtype().clone()), + self.list_size, + self.nullability, + ) } /// Returns the length of the array. pub fn len(&self) -> usize { - self.len + if self.list_size == 0 { + self.degenerate_len + } else { + self.elements().len() / self.list_size as usize + } } /// Returns `true` if the array is empty. @@ -242,7 +245,7 @@ impl FixedSizeListData { /// Returns the validity of the array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) } /// Returns the validity as a [`Mask`](vortex_mask::Mask). @@ -266,10 +269,13 @@ impl FixedSizeListData { impl Array { /// Creates a new `FixedSizeListArray`. pub fn new(elements: ArrayRef, list_size: u32, validity: Validity, len: usize) -> Self { + let dtype = DType::FixedSizeList( + Arc::new(elements.dtype().clone()), + list_size, + validity.nullability(), + ); let data = FixedSizeListData::new(elements, list_size, validity, len); - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(FixedSizeList, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(FixedSizeList, dtype, len, data)) .vortex_expect("FixedSizeListData is always valid") } @@ -280,10 +286,13 @@ impl Array { validity: Validity, len: usize, ) -> VortexResult { + let dtype = DType::FixedSizeList( + Arc::new(elements.dtype().clone()), + list_size, + validity.nullability(), + ); let data = FixedSizeListData::try_new(elements, list_size, validity, len)?; - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(FixedSizeList, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(FixedSizeList, dtype, len, data)) } /// Creates a new `FixedSizeListArray` without validation. @@ -297,21 +306,24 @@ impl Array { validity: Validity, len: usize, ) -> Self { + let dtype = DType::FixedSizeList( + Arc::new(elements.dtype().clone()), + list_size, + validity.nullability(), + ); let data = unsafe { FixedSizeListData::new_unchecked(elements, list_size, validity, len) }; - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(FixedSizeList, dtype, len, data)) - .vortex_expect("FixedSizeListData is always valid") + Array::try_from_parts(ArrayParts::new(FixedSizeList, dtype, len, data)) + .vortex_expect("FixedSizeListData is always valid") } } impl FixedSizeListData { pub fn fixed_size_list_elements_at(&self, index: usize) -> VortexResult { debug_assert!( - index < self.len, + index < self.len(), "index {} out of bounds: the len is {}", index, - self.len, + self.len(), ); debug_assert!(self.validity().is_valid(index).unwrap_or(false)); diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs index 5f5b6d12b51..2d0234b169d 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs @@ -27,7 +27,6 @@ use crate::dtype::DType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; mod kernel; @@ -106,6 +105,25 @@ impl VTable for FixedSizeList { Ok(Some(vec![])) } + fn validate(&self, data: &FixedSizeListData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.len() == len, + "FixedSizeListArray length {} does not match outer length {}", + data.len(), + len + ); + + let actual_dtype = data.dtype(); + vortex_ensure!( + &actual_dtype == dtype, + "FixedSizeListArray dtype {} does not match outer dtype {}", + actual_dtype, + dtype + ); + + Ok(()) + } + fn deserialize( &self, dtype: &DType, diff --git a/vortex-array/src/arrays/list/array.rs b/vortex-array/src/arrays/list/array.rs index 5cdf75f7945..25fe87e145d 100644 --- a/vortex-array/src/arrays/list/array.rs +++ b/vortex-array/src/arrays/list/array.rs @@ -16,7 +16,7 @@ use crate::LEGACY_SESSION; use crate::VortexSessionExecute; use crate::aggregate_fn::fns::min_max::min_max; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::ConstantArray; @@ -24,11 +24,11 @@ use crate::arrays::List; use crate::arrays::Primitive; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; +use crate::dtype::Nullability; use crate::dtype::NativePType; use crate::match_each_integer_ptype; use crate::match_each_native_ptype; use crate::scalar_fn::fns::operators::Operator; -use crate::stats::ArrayStats; use crate::validity::Validity; /// The elements data array containing all list elements concatenated together. @@ -94,9 +94,8 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["elements", "offsets", "validi /// ``` #[derive(Clone, Debug)] pub struct ListData { - pub(super) dtype: DType, + pub(super) nullability: Nullability, pub(super) slots: Vec>, - pub(super) stats_set: ArrayStats, } pub struct ListArrayParts { @@ -161,9 +160,8 @@ impl ListData { let validity_slot = validity_to_child(&validity, len); Self { - dtype: DType::List(Arc::new(elements.dtype().clone()), validity.nullability()), + nullability: validity.nullability(), slots: vec![Some(elements), Some(offsets), validity_slot], - stats_set: Default::default(), } } @@ -256,7 +254,7 @@ impl ListData { pub fn into_parts(mut self) -> ListArrayParts { let validity = self.validity(); ListArrayParts { - dtype: self.dtype, + dtype: self.dtype(), elements: self.slots[ELEMENTS_SLOT] .take() .vortex_expect("ListArray elements slot"), @@ -268,8 +266,8 @@ impl ListData { } /// Returns the dtype of the array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + DType::List(Arc::new(self.elements().dtype().clone()), self.nullability) } /// Returns the length of the array. @@ -285,7 +283,7 @@ impl ListData { /// Returns the validity of the array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) } /// Returns the validity as a [`Mask`](vortex_mask::Mask). @@ -341,11 +339,8 @@ impl ListData { } /// Returns the element dtype of the list array. - pub fn element_dtype(&self) -> &Arc { - match &self.dtype { - DType::List(element_dtype, _) => element_dtype, - _ => vortex_panic!("ListArray has invalid dtype {}", self.dtype), - } + pub fn element_dtype(&self) -> &DType { + self.elements().dtype() } /// Returns the elements array. @@ -364,10 +359,10 @@ impl ListData { impl Array { /// Creates a new `ListArray`. pub fn new(elements: ArrayRef, offsets: ArrayRef, validity: Validity) -> Self { + let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); + let len = offsets.len().saturating_sub(1); let data = ListData::new(elements, offsets, validity); - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(List, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(List, dtype, len, data)) .vortex_expect("ListData is always valid") } @@ -377,10 +372,10 @@ impl Array { offsets: ArrayRef, validity: Validity, ) -> VortexResult { + let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); + let len = offsets.len().saturating_sub(1); let data = ListData::try_new(elements, offsets, validity)?; - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(List, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(List, dtype, len, data)) } /// Creates a new `ListArray` without validation. @@ -389,10 +384,10 @@ impl Array { /// /// See [`ListData::new_unchecked`]. pub unsafe fn new_unchecked(elements: ArrayRef, offsets: ArrayRef, validity: Validity) -> Self { + let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); + let len = offsets.len().saturating_sub(1); let data = unsafe { ListData::new_unchecked(elements, offsets, validity) }; - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(List, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(List, dtype, len, data)) .vortex_expect("ListData is always valid") } } @@ -404,9 +399,9 @@ impl ListData { elements = elements.to_canonical()?.compact()?.into_array(); } else if recurse && let Some(child_list_array) = elements.as_opt::() { let data = child_list_array.reset_offsets(recurse)?; - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); - elements = Array::try_from_parts(ArrayNew::new(List, dtype, len, data))?.into_array(); + elements = Array::try_from_parts(ArrayParts::new(List, dtype, len, data))?.into_array(); } let offsets = self.offsets(); diff --git a/vortex-array/src/arrays/list/vtable/mod.rs b/vortex-array/src/arrays/list/vtable/mod.rs index 65bc4b92fe9..52683a04e43 100644 --- a/vortex-array/src/arrays/list/vtable/mod.rs +++ b/vortex-array/src/arrays/list/vtable/mod.rs @@ -32,7 +32,6 @@ use crate::hash::ArrayHash; use crate::metadata::DeserializeMetadata; use crate::metadata::SerializeMetadata; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; mod operations; @@ -97,6 +96,25 @@ impl VTable for List { .serialize())) } + fn validate(&self, data: &ListData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.len() == len, + "ListArray length {} does not match outer length {}", + data.len(), + len + ); + + let actual_dtype = data.dtype(); + vortex_ensure!( + &actual_dtype == dtype, + "ListArray dtype {} does not match outer dtype {}", + actual_dtype, + dtype + ); + + Ok(()) + } + fn deserialize( &self, dtype: &DType, diff --git a/vortex-array/src/arrays/listview/array.rs b/vortex-array/src/arrays/listview/array.rs index a2881f374c1..eb3fc9d6ccf 100644 --- a/vortex-array/src/arrays/listview/array.rs +++ b/vortex-array/src/arrays/listview/array.rs @@ -13,7 +13,7 @@ use vortex_error::vortex_err; use crate::ArrayRef; use crate::ToCanonical; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::ListView; @@ -22,8 +22,8 @@ use crate::arrays::PrimitiveArray; use crate::arrays::bool; use crate::dtype::DType; use crate::dtype::IntegerPType; +use crate::dtype::Nullability; use crate::match_each_integer_ptype; -use crate::stats::ArrayStats; use crate::validity::Validity; /// The `elements` data array, where each list scalar is a _slice_ of the `elements` array, and @@ -108,10 +108,8 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["elements", "offsets", "sizes" /// [`ListArray`]: crate::arrays::ListArray #[derive(Clone, Debug)] pub struct ListViewData { - /// The [`DType`] of the list array. - /// - /// This type **must** be the variant [`DType::List`]. - pub(super) dtype: DType, + /// The nullability of the list array. + pub(super) nullability: Nullability, /// Slots holding [elements, offsets, sizes]. pub(super) slots: Vec>, @@ -126,8 +124,6 @@ pub struct ListViewData { /// process which must rebuild the array from scratch. is_zero_copy_to_list: bool, - /// The stats for this array. - pub(super) stats_set: ArrayStats, } pub struct ListViewArrayParts { @@ -176,10 +172,9 @@ impl ListViewData { let validity_slot = validity_to_child(&validity, len); Ok(Self { - dtype: DType::List(Arc::new(elements.dtype().clone()), validity.nullability()), + nullability: validity.nullability(), slots: vec![Some(elements), Some(offsets), Some(sizes), validity_slot], is_zero_copy_to_list: false, - stats_set: Default::default(), }) } @@ -217,10 +212,9 @@ impl ListViewData { let validity_slot = validity_to_child(&validity, len); Self { - dtype: DType::List(Arc::new(elements.dtype().clone()), validity.nullability()), + nullability: validity.nullability(), slots: vec![Some(elements), Some(offsets), Some(sizes), validity_slot], is_zero_copy_to_list: false, - stats_set: Default::default(), } } @@ -346,9 +340,8 @@ impl ListViewData { pub fn into_parts(mut self) -> ListViewArrayParts { let validity = self.validity(); - let dtype = self.dtype.into_list_element_opt().vortex_expect("is list"); ListViewArrayParts { - elements_dtype: dtype, + elements_dtype: Arc::new(self.elements().dtype().clone()), elements: self.slots[ELEMENTS_SLOT] .take() .vortex_expect("ListViewArray elements slot"), @@ -363,8 +356,8 @@ impl ListViewData { } /// Returns the dtype of the array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + DType::List(Arc::new(self.elements().dtype().clone()), self.nullability) } /// Returns the length of the array. @@ -381,7 +374,7 @@ impl ListViewData { /// Returns the validity of the array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) } /// Returns the validity as a [`Mask`](vortex_mask::Mask). @@ -486,10 +479,10 @@ impl ListViewData { impl Array { /// Creates a new `ListViewArray`. pub fn new(elements: ArrayRef, offsets: ArrayRef, sizes: ArrayRef, validity: Validity) -> Self { + let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); + let len = offsets.len(); let data = ListViewData::new(elements, offsets, sizes, validity); - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(ListView, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(ListView, dtype, len, data)) .vortex_expect("ListViewData is always valid") } @@ -500,10 +493,10 @@ impl Array { sizes: ArrayRef, validity: Validity, ) -> VortexResult { + let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); + let len = offsets.len(); let data = ListViewData::try_new(elements, offsets, sizes, validity)?; - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(ListView, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(ListView, dtype, len, data)) } /// Creates a new `ListViewArray` without validation. @@ -517,11 +510,11 @@ impl Array { sizes: ArrayRef, validity: Validity, ) -> Self { + let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); + let len = offsets.len(); let data = unsafe { ListViewData::new_unchecked(elements, offsets, sizes, validity) }; - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(ListView, dtype, len, data)) - .vortex_expect("ListViewData is always valid") + Array::try_from_parts(ArrayParts::new(ListView, dtype, len, data)) + .vortex_expect("ListViewData is always valid") } /// Mark whether this list view can be zero-copy converted to a list. @@ -533,7 +526,7 @@ impl Array { let dtype = self.dtype().clone(); let len = self.len(); let data = unsafe { self.into_data().with_zero_copy_to_list(is_zctl) }; - Array::try_from_parts(ArrayNew::new(ListView, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(ListView, dtype, len, data)) .vortex_expect("data is always valid") } } diff --git a/vortex-array/src/arrays/listview/conversion.rs b/vortex-array/src/arrays/listview/conversion.rs index 6eef7440064..a24b541c1d8 100644 --- a/vortex-array/src/arrays/listview/conversion.rs +++ b/vortex-array/src/arrays/listview/conversion.rs @@ -10,7 +10,7 @@ use crate::ExecutionCtx; use crate::IntoArray; use crate::ToCanonical; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::arrays::ExtensionArray; use crate::arrays::FixedSizeListArray; use crate::arrays::List; @@ -41,7 +41,7 @@ pub fn list_view_from_list(list: ListArray, ctx: &mut ExecutionCtx) -> VortexRes let data = list.reset_offsets(false).vortex_expect("This can't fail"); let dtype = data.dtype().clone(); let len = data.len(); - let list: ListArray = Array::try_from_parts(ArrayNew::new(List, dtype, len, data))?; + let list: ListArray = Array::try_from_parts(ArrayParts::new(List, dtype, len, data))?; let list_offsets = list.offsets().clone(); diff --git a/vortex-array/src/arrays/listview/vtable/mod.rs b/vortex-array/src/arrays/listview/vtable/mod.rs index 2c3ceffe9fa..cef458ce078 100644 --- a/vortex-array/src/arrays/listview/vtable/mod.rs +++ b/vortex-array/src/arrays/listview/vtable/mod.rs @@ -29,7 +29,6 @@ use crate::dtype::PType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; mod operations; @@ -98,6 +97,25 @@ impl VTable for ListView { .serialize())) } + fn validate(&self, data: &ListViewData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.len() == len, + "ListViewArray length {} does not match outer length {}", + data.len(), + len + ); + + let actual_dtype = data.dtype(); + vortex_ensure!( + &actual_dtype == dtype, + "ListViewArray dtype {} does not match outer dtype {}", + actual_dtype, + dtype + ); + + Ok(()) + } + fn deserialize( &self, dtype: &DType, diff --git a/vortex-array/src/arrays/listview/vtable/operations.rs b/vortex-array/src/arrays/listview/vtable/operations.rs index 4a15c3dd8d0..b32bb87de1f 100644 --- a/vortex-array/src/arrays/listview/vtable/operations.rs +++ b/vortex-array/src/arrays/listview/vtable/operations.rs @@ -26,7 +26,7 @@ impl OperationsVTable for ListView { Ok(Scalar::list( Arc::new(list.dtype().clone()), children, - array.dtype.nullability(), + array.dtype().nullability(), )) } } diff --git a/vortex-array/src/arrays/masked/array.rs b/vortex-array/src/arrays/masked/array.rs index cdda1d10e44..1da29d7a35e 100644 --- a/vortex-array/src/arrays/masked/array.rs +++ b/vortex-array/src/arrays/masked/array.rs @@ -7,12 +7,12 @@ use vortex_error::vortex_bail; use crate::ArrayRef; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Masked; use crate::dtype::DType; -use crate::stats::ArrayStats; +use crate::dtype::Nullability; use crate::validity::Validity; /// The underlying child array being masked. @@ -25,8 +25,6 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child", "validity"]; #[derive(Clone, Debug)] pub struct MaskedData { pub(super) slots: Vec>, - pub(super) dtype: DType, - pub(super) stats: ArrayStats, } impl MaskedData { @@ -47,20 +45,17 @@ impl MaskedData { // MaskedArray's nullability is determined solely by its validity, not the child's dtype. // The child can have nullable dtype but must not have any actual null values. - let dtype = child.dtype().as_nullable(); let len = child.len(); let validity_slot = validity_to_child(&validity, len); Ok(Self { slots: vec![Some(child), validity_slot], - dtype, - stats: ArrayStats::default(), }) } /// Returns the dtype of the array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + self.child().dtype().as_nullable() } /// Returns the length of the array. @@ -76,7 +71,7 @@ impl MaskedData { /// Returns the validity of the array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], Nullability::Nullable) } /// Returns the validity as a [`Mask`](vortex_mask::Mask). @@ -97,6 +92,6 @@ impl Array { let dtype = child.dtype().as_nullable(); let len = child.len(); let data = MaskedData::try_new(child, validity)?; - Array::try_from_parts(ArrayNew::new(Masked, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Masked, dtype, len, data)) } } diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index 809a301d534..269fb626a92 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -33,7 +33,6 @@ use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::scalar::Scalar; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; vtable!(Masked, Masked, MaskedData); @@ -55,6 +54,15 @@ impl VTable for Masked { Self::ID } + fn validate(&self, data: &MaskedData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!(data.child().len() == len, "MaskedArray child length mismatch"); + vortex_ensure!( + data.dtype() == *dtype, + "MaskedArray dtype does not match child and validity" + ); + Ok(()) + } + fn array_hash(array: &MaskedData, state: &mut H, precision: Precision) { array.child().array_hash(state, precision); array.validity().array_hash(state, precision); @@ -84,7 +92,8 @@ impl VTable for Masked { fn deserialize( &self, dtype: &DType, - len: usize, metadata: &[u8], + len: usize, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, @@ -180,8 +189,8 @@ mod tests { use crate::arrays::MaskedArray; use crate::arrays::PrimitiveArray; use crate::dtype::Nullability; - use crate::serde::ArrayParts; use crate::serde::SerializeOptions; + use crate::serde::SerializedArray; use crate::validity::Validity; #[rstest] @@ -221,7 +230,7 @@ mod tests { } let concat = concat.freeze(); - let parts = ArrayParts::try_from(concat).unwrap(); + let parts = SerializedArray::try_from(concat).unwrap(); let decoded = parts .decode( &dtype, diff --git a/vortex-array/src/arrays/null/mod.rs b/vortex-array/src/arrays/null/mod.rs index b240a7dc2d0..2264116d584 100644 --- a/vortex-array/src/arrays/null/mod.rs +++ b/vortex-array/src/arrays/null/mod.rs @@ -14,7 +14,7 @@ use crate::ExecutionResult; use crate::Precision; use crate::array::Array; use crate::array::ArrayId; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::array::VTable; @@ -24,7 +24,6 @@ use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::scalar::Scalar; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; @@ -44,6 +43,11 @@ impl VTable for Null { Self::ID } + fn validate(&self, _data: &NullData, dtype: &DType, _len: usize) -> VortexResult<()> { + vortex_ensure!(*dtype == DType::Null, "NullArray dtype must be DType::Null"); + Ok(()) + } + fn array_hash(_array: &NullData, _state: &mut H, _precision: Precision) { // len and dtype are hashed by ArrayInner; NullData has no additional fields. } @@ -90,14 +94,15 @@ impl VTable for Null { fn deserialize( &self, _dtype: &DType, - len: usize, metadata: &[u8], + _len: usize, + metadata: &[u8], _buffers: &[BufferHandle], _children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { ::deserialize(metadata)?; - Ok(NullData::new(len)) + Ok(NullData::new()) } fn reduce_parent( @@ -142,9 +147,7 @@ impl VTable for Null { /// ``` #[derive(Clone, Debug)] pub struct NullData { - len: usize, slots: Vec>, - stats_set: ArrayStats, } #[derive(Clone, Debug)] @@ -156,33 +159,14 @@ impl Null { impl Array { pub fn new(len: usize) -> Self { - Array::try_from_parts(ArrayNew::new(Null, DType::Null, len, NullData::new(len))) + Array::try_from_parts(ArrayParts::new(Null, DType::Null, len, NullData::new())) .vortex_expect("NullData is always valid") } } impl NullData { - pub fn new(len: usize) -> Self { - Self { - len, - slots: vec![], - stats_set: Default::default(), - } - } - - /// Returns the dtype of the array (always [`DType::Null`]). - pub fn dtype(&self) -> &DType { - &DType::Null - } - - /// Returns the length of the array. - pub fn len(&self) -> usize { - self.len - } - - /// Returns `true` if the array is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 + pub fn new() -> Self { + Self { slots: vec![] } } } impl OperationsVTable for Null { diff --git a/vortex-array/src/arrays/patched/array.rs b/vortex-array/src/arrays/patched/array.rs index 4366c2fae2e..98460c46a6c 100644 --- a/vortex-array/src/arrays/patched/array.rs +++ b/vortex-array/src/arrays/patched/array.rs @@ -15,9 +15,9 @@ use crate::Canonical; use crate::ExecutionCtx; use crate::IntoArray; use crate::array::Array; -use crate::array::ArrayNew; -use crate::arrays::PrimitiveArray; +use crate::array::ArrayParts; use crate::arrays::Patched; +use crate::arrays::PrimitiveArray; use crate::arrays::patched::TransposedPatches; use crate::arrays::patched::patch_lanes; use crate::buffer::BufferHandle; @@ -138,9 +138,11 @@ impl IntoArray for PatchedArray { let dtype = self.base_array().dtype().clone(); let len = self.len; let stats = self.stats_set.clone(); - Array::::try_from_parts(ArrayNew::new(Patched, dtype, len, self).with_stats(stats)) - .vortex_expect("PatchedArray is always valid") - .into_array() + Array::::try_from_parts( + ArrayParts::new(Patched, dtype, len, self).with_stats(stats), + ) + .vortex_expect("PatchedArray is always valid") + .into_array() } } diff --git a/vortex-array/src/arrays/patched/vtable/mod.rs b/vortex-array/src/arrays/patched/vtable/mod.rs index 671afe60291..64a5d300917 100644 --- a/vortex-array/src/arrays/patched/vtable/mod.rs +++ b/vortex-array/src/arrays/patched/vtable/mod.rs @@ -134,12 +134,14 @@ impl VTable for Patched { } fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some(ProstMetadata(PatchedMetadata { - n_patches: u32::try_from(array.patch_indices().len())?, - n_lanes: u32::try_from(array.n_lanes)?, - offset: u32::try_from(array.offset)?, - }) - .serialize())) + Ok(Some( + ProstMetadata(PatchedMetadata { + n_patches: u32::try_from(array.patch_indices().len())?, + n_lanes: u32::try_from(array.n_lanes)?, + offset: u32::try_from(array.offset)?, + }) + .serialize(), + )) } fn deserialize( @@ -376,8 +378,8 @@ mod tests { use crate::assert_arrays_eq; use crate::builders::builder_with_capacity; use crate::patches::Patches; - use crate::serde::ArrayParts; use crate::serde::SerializeOptions; + use crate::serde::SerializedArray; use crate::validity::Validity; #[test] @@ -603,7 +605,7 @@ mod tests { } let concat = concat.freeze(); - let parts = ArrayParts::try_from(concat).unwrap(); + let parts = SerializedArray::try_from(concat).unwrap(); let decoded = parts .decode( &dtype, diff --git a/vortex-array/src/arrays/primitive/array/mod.rs b/vortex-array/src/arrays/primitive/array/mod.rs index b6a592ae6ca..b58cb7e68a0 100644 --- a/vortex-array/src/arrays/primitive/array/mod.rs +++ b/vortex-array/src/arrays/primitive/array/mod.rs @@ -15,14 +15,13 @@ use vortex_error::vortex_panic; use crate::ToCanonical; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::arrays::Primitive; use crate::dtype::DType; use crate::dtype::NativePType; use crate::dtype::Nullability; use crate::dtype::PType; use crate::match_each_native_ptype; -use crate::stats::ArrayStats; use crate::validity::Validity; mod accessor; @@ -77,9 +76,9 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; #[derive(Clone, Debug)] pub struct PrimitiveData { pub(super) slots: Vec>, - pub(super) dtype: DType, + pub(super) ptype: PType, + pub(super) nullability: Nullability, pub(super) buffer: BufferHandle, - pub(super) stats_set: ArrayStats, } pub struct PrimitiveArrayParts { @@ -108,12 +107,11 @@ impl PrimitiveData { ) -> Self { let len = handle.len() / ptype.byte_width(); let slots = Self::make_slots(&validity, len); - let dtype = DType::Primitive(ptype, validity.nullability()); Self { slots, buffer: handle, - dtype, - stats_set: ArrayStats::default(), + ptype, + nullability: validity.nullability(), } } @@ -164,12 +162,11 @@ impl PrimitiveData { let len = buffer.len(); let slots = Self::make_slots(&validity, len); - let dtype = DType::Primitive(T::PTYPE, validity.nullability()); Self { slots, - dtype, + ptype: T::PTYPE, + nullability: validity.nullability(), buffer: BufferHandle::new_host(buffer.into_byte_buffer()), - stats_set: Default::default(), } } @@ -198,10 +195,10 @@ impl PrimitiveData { impl Array { pub fn empty(nullability: Nullability) -> Self { + let dtype = DType::Primitive(T::PTYPE, nullability); + let len = 0; let data = PrimitiveData::empty::(nullability); - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(Primitive, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Primitive, dtype, len, data)) .vortex_expect("PrimitiveData is always valid") } @@ -211,19 +208,20 @@ impl Array { /// /// Panics if the provided components do not satisfy the invariants. pub fn new(buffer: impl Into>, validity: Validity) -> Self { + let buffer = buffer.into(); + let dtype = DType::Primitive(T::PTYPE, validity.nullability()); + let len = buffer.len(); let data = PrimitiveData::new(buffer, validity); - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(Primitive, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Primitive, dtype, len, data)) .vortex_expect("PrimitiveData is always valid") } /// Constructs a new `PrimitiveArray`. pub fn try_new(buffer: Buffer, validity: Validity) -> VortexResult { + let dtype = DType::Primitive(T::PTYPE, validity.nullability()); + let len = buffer.len(); let data = PrimitiveData::try_new(buffer, validity)?; - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(Primitive, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Primitive, dtype, len, data)) } /// Creates a new `PrimitiveArray` without validation. @@ -232,10 +230,10 @@ impl Array { /// /// See [`PrimitiveData::new_unchecked`]. pub unsafe fn new_unchecked(buffer: Buffer, validity: Validity) -> Self { + let dtype = DType::Primitive(T::PTYPE, validity.nullability()); + let len = buffer.len(); let data = unsafe { PrimitiveData::new_unchecked(buffer, validity) }; - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(Primitive, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Primitive, dtype, len, data)) .vortex_expect("PrimitiveData is always valid") } @@ -249,28 +247,28 @@ impl Array { ptype: PType, validity: Validity, ) -> Self { + let dtype = DType::Primitive(ptype, validity.nullability()); + let len = handle.len() / ptype.byte_width(); let data = unsafe { PrimitiveData::new_unchecked_from_handle(handle, ptype, validity) }; - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(Primitive, dtype, len, data)) - .vortex_expect("PrimitiveData is always valid") + Array::try_from_parts(ArrayParts::new(Primitive, dtype, len, data)) + .vortex_expect("PrimitiveData is always valid") } /// Creates a new `PrimitiveArray` from a [`BufferHandle`]. pub fn from_buffer_handle(handle: BufferHandle, ptype: PType, validity: Validity) -> Self { + let dtype = DType::Primitive(ptype, validity.nullability()); + let len = handle.len() / ptype.byte_width(); let data = PrimitiveData::from_buffer_handle(handle, ptype, validity); - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(Primitive, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Primitive, dtype, len, data)) .vortex_expect("PrimitiveData is always valid") } /// Creates a new `PrimitiveArray` from a [`ByteBuffer`]. pub fn from_byte_buffer(buffer: ByteBuffer, ptype: PType, validity: Validity) -> Self { + let dtype = DType::Primitive(ptype, validity.nullability()); + let len = buffer.len() / ptype.byte_width(); let data = PrimitiveData::from_byte_buffer(buffer, ptype, validity); - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(Primitive, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Primitive, dtype, len, data)) .vortex_expect("PrimitiveData is always valid") } @@ -281,16 +279,12 @@ impl Array { validity: Validity, n_rows: usize, ) -> Self { - let data = PrimitiveData::from_values_byte_buffer( - valid_elems_buffer, - ptype, - validity, - n_rows, - ); - let dtype = data.dtype().clone(); - let len = data.len(); - Array::try_from_parts(ArrayNew::new(Primitive, dtype, len, data)) - .vortex_expect("PrimitiveData is always valid") + let dtype = DType::Primitive(ptype, validity.nullability()); + let len = n_rows; + let data = + PrimitiveData::from_values_byte_buffer(valid_elems_buffer, ptype, validity, n_rows); + Array::try_from_parts(ArrayParts::new(Primitive, dtype, len, data)) + .vortex_expect("PrimitiveData is always valid") } /// Validates the components that would be used to create a `PrimitiveArray`. @@ -314,8 +308,8 @@ impl PrimitiveData { impl PrimitiveData { /// Returns the dtype of the array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + DType::Primitive(self.ptype, self.nullability) } /// Returns the length of the array. @@ -331,7 +325,7 @@ impl PrimitiveData { /// Reconstructs the validity from the slot state. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) } /// Returns the validity as a [`Mask`](vortex_mask::Mask). @@ -340,7 +334,7 @@ impl PrimitiveData { } pub fn ptype(&self) -> PType { - self.dtype().as_ptype() + self.ptype } /// Get access to the buffer handle backing the array. @@ -349,14 +343,13 @@ impl PrimitiveData { } pub fn from_buffer_handle(handle: BufferHandle, ptype: PType, validity: Validity) -> Self { - let dtype = DType::Primitive(ptype, validity.nullability()); let len = handle.len() / ptype.byte_width(); let slots = Self::make_slots(&validity, len); Self { slots, buffer: handle, - dtype, - stats_set: ArrayStats::default(), + ptype, + nullability: validity.nullability(), } } diff --git a/vortex-array/src/arrays/primitive/compute/between.rs b/vortex-array/src/arrays/primitive/compute/between.rs index 09ea69e6d16..251aa9c9e7c 100644 --- a/vortex-array/src/arrays/primitive/compute/between.rs +++ b/vortex-array/src/arrays/primitive/compute/between.rs @@ -33,7 +33,7 @@ impl BetweenKernel for Primitive { // null values let nullability = - arr.dtype.nullability() | lower.dtype().nullability() | upper.dtype().nullability(); + arr.dtype().nullability() | lower.dtype().nullability() | upper.dtype().nullability(); Ok(Some(match_each_native_ptype!(arr.ptype(), |P| { between_impl::

( diff --git a/vortex-array/src/arrays/primitive/vtable/mod.rs b/vortex-array/src/arrays/primitive/vtable/mod.rs index 97661e8d04c..4221a3d8237 100644 --- a/vortex-array/src/arrays/primitive/vtable/mod.rs +++ b/vortex-array/src/arrays/primitive/vtable/mod.rs @@ -37,7 +37,6 @@ use crate::arrays::primitive::array::SLOT_NAMES; use crate::arrays::primitive::compute::rules::RULES; use crate::hash::ArrayEq; use crate::hash::ArrayHash; -use crate::stats::ArrayStats; vtable!(Primitive, Primitive, PrimitiveData); @@ -83,10 +82,30 @@ impl VTable for Primitive { Ok(Some(vec![])) } + fn validate(&self, data: &PrimitiveData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.len() == len, + "PrimitiveArray length {} does not match outer length {}", + data.len(), + len + ); + + let actual_dtype = data.dtype(); + vortex_ensure!( + &actual_dtype == dtype, + "PrimitiveArray dtype {} does not match outer dtype {}", + actual_dtype, + dtype + ); + + Ok(()) + } + fn deserialize( &self, dtype: &DType, - len: usize, metadata: &[u8], + len: usize, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, @@ -199,8 +218,8 @@ mod tests { use crate::LEGACY_SESSION; use crate::arrays::PrimitiveArray; use crate::assert_arrays_eq; - use crate::serde::ArrayParts; use crate::serde::SerializeOptions; + use crate::serde::SerializedArray; use crate::validity::Validity; #[test] @@ -223,7 +242,7 @@ mod tests { for buf in serialized { concat.extend_from_slice(buf.as_ref()); } - let parts = ArrayParts::try_from(concat.freeze()).unwrap(); + let parts = SerializedArray::try_from(concat.freeze()).unwrap(); let decoded = parts .decode( &dtype, diff --git a/vortex-array/src/arrays/scalar_fn/array.rs b/vortex-array/src/arrays/scalar_fn/array.rs index 1227f779603..4c394ddf009 100644 --- a/vortex-array/src/arrays/scalar_fn/array.rs +++ b/vortex-array/src/arrays/scalar_fn/array.rs @@ -7,21 +7,16 @@ use vortex_error::vortex_ensure; use crate::ArrayRef; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::arrays::ScalarFnVTable; -use crate::dtype::DType; use crate::scalar_fn::ScalarFnRef; -use crate::stats::ArrayStats; // ScalarFnArray has a variable number of slots (one per child) #[derive(Clone, Debug)] pub struct ScalarFnData { - pub(super) vtable: ScalarFnVTable, - pub(super) dtype: DType, - pub(super) len: usize, + pub(super) scalar_fn: ScalarFnRef, pub(super) slots: Vec>, - pub(super) stats: ArrayStats, } impl ScalarFnData { @@ -31,9 +26,6 @@ impl ScalarFnData { children: Vec, len: usize, ) -> VortexResult { - let arg_dtypes: Vec<_> = children.iter().map(|c| c.dtype().clone()).collect(); - let dtype = scalar_fn.return_dtype(&arg_dtypes)?; - vortex_ensure!( children.iter().all(|c| c.len() == len), "ScalarFnArray must have children equal to the array length" @@ -41,35 +33,14 @@ impl ScalarFnData { let slots = children.into_iter().map(Some).collect(); - Ok(Self { - vtable: ScalarFnVTable { scalar_fn }, - dtype, - len, - slots, - stats: Default::default(), - }) - } - - /// Returns the dtype of the array. - pub fn dtype(&self) -> &DType { - &self.dtype - } - - /// Returns the length of the array. - pub fn len(&self) -> usize { - self.len - } - - /// Returns `true` if the array is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 + Ok(Self { scalar_fn, slots }) } /// Get the scalar function bound to this array. #[allow(clippy::same_name_method)] #[inline(always)] pub fn scalar_fn(&self) -> &ScalarFnRef { - &self.vtable.scalar_fn + &self.scalar_fn } /// Get a child array by index. @@ -117,10 +88,10 @@ impl Array { children: Vec, len: usize, ) -> VortexResult { - let data = ScalarFnData::try_new(scalar_fn, children, len)?; - let dtype = data.dtype().clone(); - let len = data.len(); - let vtable = data.vtable.clone(); - Array::try_from_parts(ArrayNew::new(vtable, dtype, len, data)) + let arg_dtypes: Vec<_> = children.iter().map(|c| c.dtype().clone()).collect(); + let dtype = scalar_fn.return_dtype(&arg_dtypes)?; + let data = ScalarFnData::try_new(scalar_fn.clone(), children, len)?; + let vtable = ScalarFnVTable { scalar_fn }; + Array::try_from_parts(ArrayParts::new(vtable, dtype, len, data)) } } diff --git a/vortex-array/src/arrays/scalar_fn/metadata.rs b/vortex-array/src/arrays/scalar_fn/metadata.rs deleted file mode 100644 index 8a0817bf089..00000000000 --- a/vortex-array/src/arrays/scalar_fn/metadata.rs +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::fmt::Debug; -use std::fmt::Formatter; - -use crate::dtype::DType; -use crate::scalar_fn::ScalarFnRef; - -#[derive(Clone)] -pub struct ScalarFnMetadata { - pub(super) scalar_fn: ScalarFnRef, - pub(super) child_dtypes: Vec, -} - -// Array tree display wrongly uses debug... -impl Debug for ScalarFnMetadata { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.scalar_fn.options()) - } -} diff --git a/vortex-array/src/arrays/scalar_fn/mod.rs b/vortex-array/src/arrays/scalar_fn/mod.rs index d19d1157d0b..778075b9fe0 100644 --- a/vortex-array/src/arrays/scalar_fn/mod.rs +++ b/vortex-array/src/arrays/scalar_fn/mod.rs @@ -2,7 +2,6 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -mod metadata; mod rules; mod vtable; diff --git a/vortex-array/src/arrays/scalar_fn/rules.rs b/vortex-array/src/arrays/scalar_fn/rules.rs index fc818258240..2c1aa67cdaa 100644 --- a/vortex-array/src/arrays/scalar_fn/rules.rs +++ b/vortex-array/src/arrays/scalar_fn/rules.rs @@ -60,7 +60,7 @@ impl ArrayReduceRule for ScalarFnPackToStructRule { StructArray::try_new( pack_options.names.clone(), array.children(), - array.len, + array.len(), validity, )? .into_array(), @@ -79,7 +79,7 @@ impl ArrayReduceRule for ScalarFnConstantRule { Ok(Some(Canonical::empty(array.dtype()).into_array())) } else { let result = array.array().scalar_at(0)?; - Ok(Some(ConstantArray::new(result, array.len).into_array())) + Ok(Some(ConstantArray::new(result, array.len()).into_array())) } } } @@ -114,7 +114,7 @@ impl ArrayReduceRule for ScalarFnAbstractReduceRule { fn reduce(&self, array: ArrayView<'_, ScalarFnVTable>) -> VortexResult> { if let Some(reduced) = array .scalar_fn() - .reduce(array.as_ref(), &ArrayReduceCtx { len: array.len })? + .reduce(array.as_ref(), &ArrayReduceCtx { len: array.len() })? { return Ok(Some( reduced diff --git a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs index e6a707b8165..9e042fa647d 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs @@ -23,7 +23,7 @@ use crate::IntoArray; use crate::Precision; use crate::array::Array; use crate::array::ArrayId; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::array::ArrayView; use crate::array::VTable; use crate::arrays::scalar_fn::array::ScalarFnData; @@ -62,6 +62,24 @@ impl VTable for ScalarFnVTable { self.scalar_fn.id() } + fn validate(&self, data: &ScalarFnData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.scalar_fn == self.scalar_fn, + "ScalarFnArray data scalar_fn does not match vtable" + ); + vortex_ensure!( + data.iter_children().all(|c| c.len() == len), + "All child arrays must have the same length as the scalar function array" + ); + + let child_dtypes = data.iter_children().map(|c| c.dtype().clone()).collect_vec(); + vortex_ensure!( + self.scalar_fn.return_dtype(&child_dtypes)? == *dtype, + "ScalarFnArray dtype does not match scalar function return dtype" + ); + Ok(()) + } + fn array_hash(array: &ScalarFnData, state: &mut H, precision: Precision) { array.scalar_fn().hash(state); for child in array.iter_children() { @@ -101,7 +119,8 @@ impl VTable for ScalarFnVTable { fn deserialize( &self, _dtype: &DType, - _len: usize, _metadata: &[u8], + _len: usize, + _metadata: &[u8], _buffers: &[BufferHandle], _children: &dyn ArrayChildren, @@ -170,16 +189,11 @@ pub trait ScalarFnArrayExt: scalar_fn::ScalarFnVTable { let dtype = scalar_fn.return_dtype(&child_dtypes)?; let data = ScalarFnData { - vtable: ScalarFnVTable { scalar_fn }, - dtype, - len, + scalar_fn: scalar_fn.clone(), slots: children.into_iter().map(Some).collect(), - stats: Default::default(), }; - let vtable = data.vtable.clone(); - let dtype = data.dtype.clone(); - let len = data.len; - Array::try_from_parts(ArrayNew::new(vtable, dtype, len, data)).map(IntoArray::into_array) + let vtable = ScalarFnVTable { scalar_fn }; + Array::try_from_parts(ArrayParts::new(vtable, dtype, len, data)).map(IntoArray::into_array) } } impl ScalarFnArrayExt for V {} diff --git a/vortex-array/src/arrays/scalar_fn/vtable/operations.rs b/vortex-array/src/arrays/scalar_fn/vtable/operations.rs index f02f0b7f9bf..c697d784ea8 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/operations.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/operations.rs @@ -44,11 +44,11 @@ impl OperationsVTable for ScalarFnVTable { debug_assert_eq!( scalar.dtype(), - &array.dtype, + array.dtype(), "Scalar function {} returned dtype {:?} but expected {:?}", array.scalar_fn(), scalar.dtype(), - array.dtype + array.dtype() ); Ok(scalar) diff --git a/vortex-array/src/arrays/shared/array.rs b/vortex-array/src/arrays/shared/array.rs index 0a8b2e59169..712a1c2a479 100644 --- a/vortex-array/src/arrays/shared/array.rs +++ b/vortex-array/src/arrays/shared/array.rs @@ -14,10 +14,9 @@ use crate::ArrayRef; use crate::Canonical; use crate::IntoArray; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::arrays::Shared; use crate::dtype::DType; -use crate::stats::ArrayStats; /// The source array that is shared and lazily computed. pub(super) const SOURCE_SLOT: usize = 0; @@ -33,18 +32,14 @@ pub struct SharedData { pub(super) slots: Vec>, cached: Arc>>, async_compute_lock: Arc>, - pub(super) dtype: DType, - pub(super) stats: ArrayStats, } impl SharedData { pub fn new(source: ArrayRef) -> Self { Self { - dtype: source.dtype().clone(), slots: vec![Some(source)], cached: Arc::new(OnceLock::new()), async_compute_lock: Arc::new(AsyncMutex::new(())), - stats: ArrayStats::default(), } } @@ -114,7 +109,7 @@ impl SharedData { /// Returns the [`DType`] of this array. pub fn dtype(&self) -> &DType { - &self.dtype + self.current_array_ref().dtype() } /// Returns `true` if this array is empty. @@ -128,16 +123,13 @@ impl Array { pub fn new(source: ArrayRef) -> Self { let dtype = source.dtype().clone(); let len = source.len(); - Array::try_from_parts(ArrayNew::new(Shared, dtype, len, SharedData::new(source))) + Array::try_from_parts(ArrayParts::new(Shared, dtype, len, SharedData::new(source))) .vortex_expect("SharedData is always valid") } } impl SharedData { pub(super) fn set_source(&mut self, source: Option) { - if let Some(ref s) = source { - self.dtype = s.dtype().clone(); - } self.slots = vec![source]; self.cached = Arc::new(OnceLock::new()); self.async_compute_lock = Arc::new(AsyncMutex::new(())); diff --git a/vortex-array/src/arrays/shared/vtable.rs b/vortex-array/src/arrays/shared/vtable.rs index 11c4714f87f..abd4723ca5d 100644 --- a/vortex-array/src/arrays/shared/vtable.rs +++ b/vortex-array/src/arrays/shared/vtable.rs @@ -8,7 +8,6 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::Canonical; -use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; @@ -25,7 +24,6 @@ use crate::dtype::DType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::scalar::Scalar; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; @@ -49,6 +47,12 @@ impl VTable for Shared { Self::ID } + fn validate(&self, data: &SharedData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_error::vortex_ensure!(data.source().dtype() == dtype, "SharedArray dtype mismatch"); + vortex_error::vortex_ensure!(data.source().len() == len, "SharedArray len mismatch"); + Ok(()) + } + fn array_hash(array: &SharedData, state: &mut H, precision: Precision) { let current = array.current_array_ref(); current.array_hash(state, precision); @@ -100,11 +104,12 @@ impl VTable for Shared { fn deserialize( &self, - dtype: &DType, - len: usize, _metadata: &[u8], + _dtype: &DType, + _len: usize, + _metadata: &[u8], _buffers: &[BufferHandle], - children: &dyn crate::serde::ArrayChildren, + _children: &dyn crate::serde::ArrayChildren, _session: &VortexSession, ) -> VortexResult { vortex_error::vortex_bail!("Shared array is not serializable") diff --git a/vortex-array/src/arrays/slice/array.rs b/vortex-array/src/arrays/slice/array.rs index 1bf3a01399b..6c2f38e9832 100644 --- a/vortex-array/src/arrays/slice/array.rs +++ b/vortex-array/src/arrays/slice/array.rs @@ -9,10 +9,9 @@ use vortex_error::vortex_panic; use crate::ArrayRef; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::arrays::Slice; use crate::dtype::DType; -use crate::stats::ArrayStats; /// The underlying child array being sliced. pub(super) const CHILD_SLOT: usize = 0; @@ -23,7 +22,6 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child"]; pub struct SliceData { pub(super) slots: Vec>, pub(super) range: Range, - pub(super) stats: ArrayStats, } pub struct SliceArrayParts { @@ -43,7 +41,6 @@ impl SliceData { Ok(Self { slots: vec![Some(child)], range, - stats: ArrayStats::default(), }) } @@ -85,7 +82,7 @@ impl Array { let len = range.len(); let dtype = child.dtype().clone(); let data = SliceData::try_new(child, range)?; - Array::try_from_parts(ArrayNew::new(Slice, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Slice, dtype, len, data)) } /// Constructs a new `SliceArray`. @@ -93,7 +90,7 @@ impl Array { let len = range.len(); let dtype = child.dtype().clone(); let data = SliceData::new(child, range); - Array::try_from_parts(ArrayNew::new(Slice, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Slice, dtype, len, data)) .vortex_expect("SliceData is always valid") } } diff --git a/vortex-array/src/arrays/slice/vtable.rs b/vortex-array/src/arrays/slice/vtable.rs index b8c59c68da4..5e0b52c91fd 100644 --- a/vortex-array/src/arrays/slice/vtable.rs +++ b/vortex-array/src/arrays/slice/vtable.rs @@ -36,7 +36,6 @@ use crate::executor::ExecutionCtx; use crate::executor::ExecutionResult; use crate::scalar::Scalar; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; @@ -95,11 +94,12 @@ impl VTable for Slice { fn deserialize( &self, - dtype: &DType, - len: usize, _metadata: &[u8], + _dtype: &DType, + _len: usize, + _metadata: &[u8], _buffers: &[BufferHandle], - children: &dyn ArrayChildren, + _children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { vortex_bail!("Slice array is not serializable") diff --git a/vortex-array/src/arrays/struct_/array.rs b/vortex-array/src/arrays/struct_/array.rs index da08ba9f78a..111bcf286fa 100644 --- a/vortex-array/src/arrays/struct_/array.rs +++ b/vortex-array/src/arrays/struct_/array.rs @@ -13,7 +13,7 @@ use vortex_error::vortex_err; use crate::ArrayRef; use crate::IntoArray; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Struct; @@ -21,7 +21,6 @@ use crate::dtype::DType; use crate::dtype::FieldName; use crate::dtype::FieldNames; use crate::dtype::StructFields; -use crate::stats::ArrayStats; use crate::validity::Validity; // StructArray has a variable number of slots: [validity?, field_0, ..., field_N] @@ -153,7 +152,6 @@ pub struct StructData { pub(super) len: usize, pub(super) dtype: DType, pub(super) slots: Vec>, - pub(super) stats_set: ArrayStats, } pub struct StructArrayParts { @@ -333,7 +331,6 @@ impl StructData { len: length, dtype: DType::Struct(dtype, validity.nullability()), slots, - stats_set: Default::default(), } } @@ -533,7 +530,7 @@ impl Array { let data = StructData::new(names, fields, length, validity); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(Struct, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) .vortex_expect("StructData is always valid") } @@ -547,7 +544,7 @@ impl Array { let data = StructData::try_new(names, fields, length, validity)?; let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(Struct, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) } /// Creates a new `StructArray` without validation. @@ -564,7 +561,7 @@ impl Array { let data = unsafe { StructData::new_unchecked(fields, dtype, length, validity) }; let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(Struct, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) .vortex_expect("StructData is always valid") } @@ -578,7 +575,7 @@ impl Array { let data = StructData::try_new_with_dtype(fields, dtype, length, validity)?; let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(Struct, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) } /// Construct a `StructArray` from named fields. @@ -586,7 +583,7 @@ impl Array { let data = StructData::from_fields(items)?; let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(Struct, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) } /// Decompose this struct array into its constituent parts. @@ -606,7 +603,7 @@ impl Array { let data = StructData::try_from_iter_with_validity(iter, validity)?; let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(Struct, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) } /// Create a `StructArray` from an iterator of (name, array) pairs. @@ -616,7 +613,7 @@ impl Array { let data = StructData::try_from_iter(iter)?; let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(Struct, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) } /// Create a fieldless `StructArray` with the given length. @@ -624,7 +621,7 @@ impl Array { let data = StructData::new_fieldless_with_len(len); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(Struct, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) .vortex_expect("StructData is always valid") } } diff --git a/vortex-array/src/arrays/struct_/vtable/mod.rs b/vortex-array/src/arrays/struct_/vtable/mod.rs index ead30676889..6034fa8e2a0 100644 --- a/vortex-array/src/arrays/struct_/vtable/mod.rs +++ b/vortex-array/src/arrays/struct_/vtable/mod.rs @@ -33,7 +33,6 @@ use crate::Precision; use crate::array::ArrayId; use crate::hash::ArrayEq; use crate::hash::ArrayHash; -use crate::stats::ArrayStats; vtable!(Struct, Struct, StructData); diff --git a/vortex-array/src/arrays/varbin/array.rs b/vortex-array/src/arrays/varbin/array.rs index 1acb47fbce8..ec7a403dbb7 100644 --- a/vortex-array/src/arrays/varbin/array.rs +++ b/vortex-array/src/arrays/varbin/array.rs @@ -12,7 +12,7 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::ToCanonical; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::VarBin; @@ -22,7 +22,6 @@ use crate::dtype::DType; use crate::dtype::IntegerPType; use crate::dtype::Nullability; use crate::match_each_integer_ptype; -use crate::stats::ArrayStats; use crate::validity::Validity; /// The offsets array defining the start/end of each variable-length binary element. @@ -37,7 +36,6 @@ pub struct VarBinData { pub(super) dtype: DType, pub(super) bytes: BufferHandle, pub(super) slots: Vec>, - pub(super) stats_set: ArrayStats, } impl VarBinData { @@ -172,7 +170,6 @@ impl VarBinData { dtype, bytes, slots: vec![Some(offsets), validity_slot], - stats_set: Default::default(), } } @@ -386,7 +383,7 @@ impl Array { let data = VarBinData::from_vec(vec, dtype); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") } @@ -401,7 +398,7 @@ impl Array { let data = VarBinData::from_iter(iter, dtype); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") } @@ -412,7 +409,7 @@ impl Array { let data = VarBinData::from_iter_nonnull(iter, dtype); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") } @@ -485,7 +482,7 @@ impl Array { let data = VarBinData::new(offsets, bytes, dtype, validity); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") } @@ -503,7 +500,7 @@ impl Array { let data = unsafe { VarBinData::new_unchecked(offsets, bytes, dtype, validity) }; let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") } @@ -518,13 +515,12 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - let data = unsafe { - VarBinData::new_unchecked_from_handle(offsets, bytes, dtype, validity) - }; + let data = + unsafe { VarBinData::new_unchecked_from_handle(offsets, bytes, dtype, validity) }; let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) - .vortex_expect("VarBinData is always valid") + Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) + .vortex_expect("VarBinData is always valid") } /// Constructs a new `VarBinArray`. @@ -537,7 +533,7 @@ impl Array { let data = VarBinData::try_new(offsets, bytes, dtype, validity)?; let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) } } @@ -620,7 +616,7 @@ impl From> for Array { let data = VarBinData::from(value); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") } } @@ -630,7 +626,7 @@ impl From>> for Array { let data = VarBinData::from(value); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") } } @@ -640,7 +636,7 @@ impl From> for Array { let data = VarBinData::from(value); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") } } @@ -650,7 +646,7 @@ impl From> for Array { let data = VarBinData::from(value); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") } } @@ -660,7 +656,7 @@ impl From>> for Array { let data = VarBinData::from(value); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") } } @@ -670,7 +666,7 @@ impl From>>> for Array { let data = VarBinData::from(value); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") } } @@ -680,7 +676,7 @@ impl From>> for Array { let data = VarBinData::from(value); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") } } @@ -690,7 +686,7 @@ impl From>> for Array { let data = VarBinData::from(value); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") } } @@ -700,7 +696,7 @@ impl<'a> FromIterator> for Array { let data = >::from_iter(iter); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect(" is always valid") } } @@ -710,7 +706,7 @@ impl FromIterator>> for Array { let data = >::from_iter(iter); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect(" is always valid") } } @@ -720,7 +716,7 @@ impl FromIterator> for Array { let data = >::from_iter(iter); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect(" is always valid") } } @@ -730,7 +726,7 @@ impl<'a> FromIterator> for Array { let data = >::from_iter(iter); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBin, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect(" is always valid") } } diff --git a/vortex-array/src/arrays/varbin/builder.rs b/vortex-array/src/arrays/varbin/builder.rs index e65bd6dc37d..8dd5066a889 100644 --- a/vortex-array/src/arrays/varbin/builder.rs +++ b/vortex-array/src/arrays/varbin/builder.rs @@ -7,8 +7,6 @@ use vortex_buffer::BufferMut; use vortex_error::vortex_panic; use crate::IntoArray; -use crate::array::VTable; -use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; use crate::arrays::VarBinArray; use crate::dtype::DType; diff --git a/vortex-array/src/arrays/varbin/vtable/mod.rs b/vortex-array/src/arrays/varbin/vtable/mod.rs index bdcbeb5f200..da69c44412f 100644 --- a/vortex-array/src/arrays/varbin/vtable/mod.rs +++ b/vortex-array/src/arrays/varbin/vtable/mod.rs @@ -41,7 +41,6 @@ use crate::Precision; use crate::arrays::varbin::compute::rules::PARENT_RULES; use crate::hash::ArrayEq; use crate::hash::ArrayHash; -use crate::stats::ArrayStats; vtable!(VarBin, VarBin, VarBinData); diff --git a/vortex-array/src/arrays/varbinview/array.rs b/vortex-array/src/arrays/varbinview/array.rs index ca9f2d990f5..61ac959f5e7 100644 --- a/vortex-array/src/arrays/varbinview/array.rs +++ b/vortex-array/src/arrays/varbinview/array.rs @@ -17,7 +17,7 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::VarBinView; @@ -27,7 +27,6 @@ use crate::builders::ArrayBuilder; use crate::builders::VarBinViewBuilder; use crate::dtype::DType; use crate::dtype::Nullability; -use crate::stats::ArrayStats; use crate::validity::Validity; /// The validity bitmap indicating which elements are non-null. @@ -100,7 +99,6 @@ pub struct VarBinViewData { pub(super) dtype: DType, pub(super) buffers: Arc<[BufferHandle]>, pub(super) views: BufferHandle, - pub(super) stats_set: ArrayStats, } pub struct VarBinViewArrayParts { @@ -269,7 +267,6 @@ impl VarBinViewData { views, buffers, dtype, - stats_set: Default::default(), } } @@ -565,7 +562,7 @@ impl Array { let data = VarBinViewData::from_iter(iter, dtype); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") } @@ -573,7 +570,7 @@ impl Array { let data = VarBinViewData::from_iter_str(iter); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") } @@ -583,7 +580,7 @@ impl Array { let data = VarBinViewData::from_iter_nullable_str(iter); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") } @@ -591,7 +588,7 @@ impl Array { let data = VarBinViewData::from_iter_bin(iter); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") } @@ -601,7 +598,7 @@ impl Array { let data = VarBinViewData::from_iter_nullable_bin(iter); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") } @@ -615,7 +612,7 @@ impl Array { let data = VarBinViewData::try_new(views, buffers, dtype, validity)?; let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) } /// Creates a new `VarBinViewArray` without validation. @@ -632,8 +629,8 @@ impl Array { let data = unsafe { VarBinViewData::new_unchecked(views, buffers, dtype, validity) }; let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) - .vortex_expect("VarBinViewData is always valid") + Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) + .vortex_expect("VarBinViewData is always valid") } /// Creates a new `VarBinViewArray` with device or host memory. @@ -646,7 +643,7 @@ impl Array { let data = VarBinViewData::new_handle(views, buffers, dtype, validity); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") } @@ -661,13 +658,11 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - let data = unsafe { - VarBinViewData::new_handle_unchecked(views, buffers, dtype, validity) - }; + let data = unsafe { VarBinViewData::new_handle_unchecked(views, buffers, dtype, validity) }; let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) - .vortex_expect("VarBinViewData is always valid") + Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) + .vortex_expect("VarBinViewData is always valid") } } @@ -702,7 +697,7 @@ impl<'a> FromIterator> for Array { let data = >::from_iter(iter); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect(" is always valid") } } @@ -712,7 +707,7 @@ impl FromIterator>> for Array { let data = >::from_iter(iter); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect(" is always valid") } } @@ -722,7 +717,7 @@ impl FromIterator> for Array { let data = >::from_iter(iter); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect(" is always valid") } } @@ -732,7 +727,7 @@ impl<'a> FromIterator> for Array { let data = >::from_iter(iter); let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayNew::new(VarBinView, dtype, len, data)) + Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect(" is always valid") } } diff --git a/vortex-array/src/arrays/varbinview/vtable/mod.rs b/vortex-array/src/arrays/varbinview/vtable/mod.rs index 78ea7d3d092..c1b997072ed 100644 --- a/vortex-array/src/arrays/varbinview/vtable/mod.rs +++ b/vortex-array/src/arrays/varbinview/vtable/mod.rs @@ -32,7 +32,6 @@ use crate::dtype::DType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; mod kernel; @@ -113,7 +112,8 @@ impl VTable for VarBinView { fn deserialize( &self, dtype: &DType, - len: usize, metadata: &[u8], + len: usize, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, @@ -215,8 +215,8 @@ mod tests { use crate::IntoArray; use crate::LEGACY_SESSION; use crate::assert_arrays_eq; - use crate::serde::ArrayParts; use crate::serde::SerializeOptions; + use crate::serde::SerializedArray; #[test] fn test_nullable_varbinview_serde_roundtrip() { @@ -241,7 +241,7 @@ mod tests { for buf in serialized { concat.extend_from_slice(buf.as_ref()); } - let parts = ArrayParts::try_from(concat.freeze()).unwrap(); + let parts = SerializedArray::try_from(concat.freeze()).unwrap(); let decoded = parts .decode( &dtype, diff --git a/vortex-array/src/arrays/variant/mod.rs b/vortex-array/src/arrays/variant/mod.rs index 96e1015d61e..78d8d316f68 100644 --- a/vortex-array/src/arrays/variant/mod.rs +++ b/vortex-array/src/arrays/variant/mod.rs @@ -9,9 +9,8 @@ pub use self::vtable::Variant; pub use self::vtable::VariantArray; use crate::ArrayRef; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::dtype::DType; -use crate::stats::ArrayStats; pub(super) const NUM_SLOTS: usize = 1; pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child"]; @@ -27,17 +26,12 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child"]; #[derive(Clone, Debug)] pub struct VariantData { pub(super) slots: Vec>, - pub(crate) stats_set: ArrayStats, } impl VariantData { /// Creates a new VariantArray. Nullability comes from the child's dtype. pub fn new(child: ArrayRef) -> Self { - let stats_set = child.statistics().to_array_stats(); - Self { - slots: vec![Some(child)], - stats_set, - } + Self { slots: vec![Some(child)] } } /// Returns the length of this array. @@ -69,7 +63,9 @@ impl Array { let dtype = child.dtype().clone(); let len = child.len(); let stats = child.statistics().to_array_stats(); - Array::try_from_parts(ArrayNew::new(Variant, dtype, len, VariantData::new(child)).with_stats(stats)) - .vortex_expect("VariantData is always valid") + Array::try_from_parts( + ArrayParts::new(Variant, dtype, len, VariantData::new(child)).with_stats(stats), + ) + .vortex_expect("VariantData is always valid") } } diff --git a/vortex-array/src/arrays/variant/vtable/mod.rs b/vortex-array/src/arrays/variant/vtable/mod.rs index bab4fded841..80855420860 100644 --- a/vortex-array/src/arrays/variant/vtable/mod.rs +++ b/vortex-array/src/arrays/variant/vtable/mod.rs @@ -27,7 +27,6 @@ use crate::arrays::variant::VariantData; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::vtable; vtable!(Variant, Variant, VariantData); diff --git a/vortex-array/src/scalar_fn/fns/select.rs b/vortex-array/src/scalar_fn/fns/select.rs index 1415aba5073..c7478824d52 100644 --- a/vortex-array/src/scalar_fn/fns/select.rs +++ b/vortex-array/src/scalar_fn/fns/select.rs @@ -19,7 +19,7 @@ use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; use crate::array::Array; -use crate::array::ArrayNew; +use crate::array::ArrayParts; use crate::arrays::Struct; use crate::arrays::StructArray; use crate::dtype::DType; @@ -165,7 +165,7 @@ impl ScalarFnVTable for Select { let dtype = result.dtype().clone(); let len = result.len(); - Array::try_from_parts(ArrayNew::new(Struct, dtype, len, result))? + Array::try_from_parts(ArrayParts::new(Struct, dtype, len, result))? .into_array() .execute(ctx) } diff --git a/vortex-array/src/serde.rs b/vortex-array/src/serde.rs index b98f1da6c38..b3d89b3b2a0 100644 --- a/vortex-array/src/serde.rs +++ b/vortex-array/src/serde.rs @@ -281,14 +281,14 @@ impl ArrayChildren for &[ArrayRef] { } } -/// [`ArrayParts`] represents a parsed but not-yet-decoded deserialized array. +/// [`SerializedArray`] represents a parsed but not-yet-decoded deserialized array. /// It contains all the information from the serialized form, without anything extra. i.e. /// it is missing a [`DType`] and `len`, and the `encoding_id` is not yet resolved to a concrete /// vtable. /// -/// An [`ArrayParts`] can be fully decoded into an [`ArrayRef`] using the `decode` function. +/// An [`SerializedArray`] can be fully decoded into an [`ArrayRef`] using the `decode` function. #[derive(Clone)] -pub struct ArrayParts { +pub struct SerializedArray { // Typed as fb::ArrayNode flatbuffer: FlatBuffer, // The location of the current fb::ArrayNode @@ -296,9 +296,9 @@ pub struct ArrayParts { buffers: Arc<[BufferHandle]>, } -impl Debug for ArrayParts { +impl Debug for SerializedArray { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ArrayParts") + f.debug_struct("SerializedArray") .field("encoding_id", &self.encoding_id()) .field("children", &(0..self.nchildren()).map(|i| self.child(i))) .field( @@ -310,8 +310,8 @@ impl Debug for ArrayParts { } } -impl ArrayParts { - /// Decode an [`ArrayParts`] into an [`ArrayRef`]. +impl SerializedArray { + /// Decode an [`SerializedArray`] into an [`ArrayRef`]. pub fn decode( &self, dtype: &DType, @@ -329,8 +329,8 @@ impl ArrayParts { .find(&encoding_id) .ok_or_else(|| vortex_err!("Unknown encoding: {}", encoding_id))?; - let children = ArrayPartsChildren { - parts: self, + let children = SerializedArrayChildren { + ser: self, ctx, session, }; @@ -402,7 +402,7 @@ impl ArrayParts { } /// Returns the nth child of the array. - pub fn child(&self, idx: usize) -> ArrayParts { + pub fn child(&self, idx: usize) -> SerializedArray { let children = self .flatbuffer() .children() @@ -505,7 +505,7 @@ impl ArrayParts { Ok((fb_buffer, flatbuffer_loc)) } - /// Create an [`ArrayParts`] from a pre-existing array tree flatbuffer and pre-resolved buffer + /// Create an [`SerializedArray`] from a pre-existing array tree flatbuffer and pre-resolved buffer /// handles. /// /// The caller is responsible for resolving buffers from whatever source (device segments, host @@ -516,14 +516,14 @@ impl ArrayParts { buffers: Vec, ) -> VortexResult { let (flatbuffer, flatbuffer_loc) = Self::validate_array_tree(array_tree)?; - Ok(ArrayParts { + Ok(SerializedArray { flatbuffer, flatbuffer_loc, buffers: buffers.into(), }) } - /// Create an [`ArrayParts`] from a raw array tree flatbuffer (metadata only). + /// Create an [`SerializedArray`] from a raw array tree flatbuffer (metadata only). /// /// This constructor creates an `ArrayParts` with no buffer data, useful for /// inspecting the metadata when the actual buffer data is not needed @@ -533,7 +533,7 @@ impl ArrayParts { /// no actual buffer data is available. pub fn from_array_tree(array_tree: impl Into) -> VortexResult { let (flatbuffer, flatbuffer_loc) = Self::validate_array_tree(array_tree)?; - Ok(ArrayParts { + Ok(SerializedArray { flatbuffer, flatbuffer_loc, buffers: Arc::new([]), @@ -545,7 +545,7 @@ impl ArrayParts { unsafe { fba::ArrayNode::follow(self.flatbuffer.as_ref(), self.flatbuffer_loc) } } - /// Returns a new [`ArrayParts`] with the given node as the root + /// Returns a new [`SerializedArray`] with the given node as the root // TODO(ngates): we may want a wrapper that avoids this clone. fn with_root(&self, root: fba::ArrayNode) -> Self { let mut this = self.clone(); @@ -553,7 +553,7 @@ impl ArrayParts { this } - /// Create an [`ArrayParts`] from a pre-existing flatbuffer (ArrayNode) and a segment containing + /// Create an [`SerializedArray`] from a pre-existing flatbuffer (ArrayNode) and a segment containing /// only the data buffers (without the flatbuffer suffix). /// /// This is used when the flatbuffer is stored separately in layout metadata (e.g., when @@ -566,7 +566,7 @@ impl ArrayParts { Self::from_flatbuffer_and_segment_with_overrides(array_tree, segment, &HashMap::new()) } - /// Create an [`ArrayParts`] from a pre-existing flatbuffer (ArrayNode) and a segment, + /// Create an [`SerializedArray`] from a pre-existing flatbuffer (ArrayNode) and a segment, /// substituting host-resident buffer overrides for specific buffer indices. /// /// Buffers whose index appears in `buffer_overrides` are resolved from the provided @@ -611,7 +611,7 @@ impl ArrayParts { }) .collect::>>()?; - Ok(ArrayParts { + Ok(SerializedArray { flatbuffer: fb_buffer, flatbuffer_loc, buffers, @@ -619,25 +619,25 @@ impl ArrayParts { } } -struct ArrayPartsChildren<'a> { - parts: &'a ArrayParts, +struct SerializedArrayChildren<'a> { + ser: &'a SerializedArray, ctx: &'a ReadContext, session: &'a VortexSession, } -impl ArrayChildren for ArrayPartsChildren<'_> { +impl ArrayChildren for SerializedArrayChildren<'_> { fn get(&self, index: usize, dtype: &DType, len: usize) -> VortexResult { - self.parts + self.ser .child(index) .decode(dtype, len, self.ctx, self.session) } fn len(&self) -> usize { - self.parts.nchildren() + self.ser.nchildren() } } -impl TryFrom for ArrayParts { +impl TryFrom for SerializedArray { type Error = VortexError; fn try_from(value: ByteBuffer) -> Result { @@ -662,7 +662,7 @@ impl TryFrom for ArrayParts { } } -impl TryFrom for ArrayParts { +impl TryFrom for SerializedArray { type Error = VortexError; fn try_from(value: BufferHandle) -> Result { diff --git a/vortex-btrblocks/src/schemes/float.rs b/vortex-btrblocks/src/schemes/float.rs index b995f0827c5..3ed6a883c44 100644 --- a/vortex-btrblocks/src/schemes/float.rs +++ b/vortex-btrblocks/src/schemes/float.rs @@ -6,7 +6,6 @@ use vortex_alp::ALP; use vortex_alp::RDEncoder; use vortex_alp::alp_encode; -use vortex_array::Array; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; @@ -147,6 +146,8 @@ impl Scheme for ALPRDScheme { }; let alp_rd = encoder.encode(stats.source()); + let dtype = alp_rd.dtype().clone(); + let right_bit_width = alp_rd.right_bit_width(); let mut alp_rd_data = alp_rd.into_data(); let patches = alp_rd_data @@ -154,8 +155,17 @@ impl Scheme for ALPRDScheme { .map(compress_patches) .transpose()?; alp_rd_data.replace_left_parts_patches(patches); - - Ok(Array::::try_from_data(alp_rd_data)?.into_array()) + let parts = alp_rd_data.into_parts(); + + Ok(vortex_alp::ALPRD::try_new( + dtype, + parts.left_parts, + parts.left_parts_dictionary, + parts.right_parts, + right_bit_width, + parts.left_parts_patches, + )? + .into_array()) } } diff --git a/vortex-btrblocks/src/schemes/integer.rs b/vortex-btrblocks/src/schemes/integer.rs index c6b4f3fee56..1f359106185 100644 --- a/vortex-btrblocks/src/schemes/integer.rs +++ b/vortex-btrblocks/src/schemes/integer.rs @@ -3,7 +3,6 @@ //! Integer compression schemes. -use vortex_array::Array; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; @@ -336,12 +335,28 @@ impl Scheme for BitPackingScheme { return Ok(stats.source().clone().into_array()); } let packed = bitpack_encode(stats.source(), bw, Some(&histogram))?; + let ptype = packed.dtype().as_ptype(); + let len = packed.len(); + let nullability = packed.dtype().nullability(); let mut packed_data = packed.into_data(); - let patches = packed_data.patches().map(compress_patches).transpose()?; + let patches = packed_data + .patches(len) + .map(compress_patches) + .transpose()?; packed_data.replace_patches(patches); - - Ok(Array::::try_from_data(packed_data)?.into_array()) + let parts = packed_data.into_parts(len, nullability); + + Ok(vortex_fastlanes::BitPacked::try_new( + parts.packed, + ptype, + parts.validity, + parts.patches, + parts.bit_width, + parts.len, + parts.offset, + )? + .into_array()) } } diff --git a/vortex-btrblocks/src/schemes/rle.rs b/vortex-btrblocks/src/schemes/rle.rs index 4da83209f39..fbc509d46cd 100644 --- a/vortex-btrblocks/src/schemes/rle.rs +++ b/vortex-btrblocks/src/schemes/rle.rs @@ -281,7 +281,6 @@ impl Scheme for RLEScheme { compressed_values, compressed_indices, compressed_offsets, - rle_array.dtype().clone(), rle_array.offset(), rle_array.len(), ) diff --git a/vortex-cuda/src/kernel/encodings/bitpacked.rs b/vortex-cuda/src/kernel/encodings/bitpacked.rs index eb7273f1ddf..4ef8b9bf1d8 100644 --- a/vortex-cuda/src/kernel/encodings/bitpacked.rs +++ b/vortex-cuda/src/kernel/encodings/bitpacked.rs @@ -103,7 +103,7 @@ where packed, patches, validity, - } = array.into_data().into_parts(); + } = BitPacked::into_parts(array); vortex_ensure!(len > 0, "Non empty array"); let offset = offset as usize; diff --git a/vortex-cuda/src/layout.rs b/vortex-cuda/src/layout.rs index bd9fd51e276..0a8efcaac5f 100644 --- a/vortex-cuda/src/layout.rs +++ b/vortex-cuda/src/layout.rs @@ -26,8 +26,8 @@ use vortex::array::expr::stats::Stat; use vortex::array::expr::stats::StatsProvider; use vortex::array::normalize::NormalizeOptions; use vortex::array::normalize::Operation; -use vortex::array::serde::ArrayParts; use vortex::array::serde::SerializeOptions; +use vortex::array::serde::SerializedArray; use vortex::array::session::ArrayRegistry; use vortex::array::stats::StatsSetRef; use vortex::buffer::BufferString; @@ -249,7 +249,7 @@ impl CudaFlatReader { async move { let segment = segment_fut.await?; - let parts = ArrayParts::from_flatbuffer_and_segment_with_overrides( + let parts = SerializedArray::from_flatbuffer_and_segment_with_overrides( array_tree, segment, &host_buffers, diff --git a/vortex-ipc/src/messages/decoder.rs b/vortex-ipc/src/messages/decoder.rs index dabbc9e47a8..d0108830aa3 100644 --- a/vortex-ipc/src/messages/decoder.rs +++ b/vortex-ipc/src/messages/decoder.rs @@ -8,7 +8,7 @@ use bytes::Buf; use flatbuffers::root; use flatbuffers::root_unchecked; use vortex_array::ArrayId; -use vortex_array::serde::ArrayParts; +use vortex_array::serde::SerializedArray; use vortex_buffer::AlignedBuf; use vortex_buffer::Alignment; use vortex_buffer::ByteBuffer; @@ -25,7 +25,7 @@ use vortex_session::registry::ReadContext; /// A message decoded from an IPC stream. #[derive(Debug)] pub enum DecoderMessage { - Array((ArrayParts, ReadContext, usize)), + Array((SerializedArray, ReadContext, usize)), Buffer(ByteBuffer), DType(FlatBuffer), } @@ -115,7 +115,7 @@ impl MessageDecoder { MessageHeader::ArrayMessage => { // We don't care about alignment here since ArrayParts will handle it. let body = bytes.copy_to_aligned(body_length, Alignment::new(1)); - let parts = ArrayParts::try_from(body)?; + let parts = SerializedArray::try_from(body)?; let header = msg .header_as_array_message() diff --git a/vortex-layout/src/display.rs b/vortex-layout/src/display.rs index c3c36a06de1..30e029ec8a3 100644 --- a/vortex-layout/src/display.rs +++ b/vortex-layout/src/display.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use futures::future::try_join_all; use termtree::Tree; -use vortex_array::serde::ArrayParts; +use vortex_array::serde::SerializedArray; use vortex_error::VortexResult; use vortex_utils::aliases::hash_map::HashMap; @@ -35,7 +35,7 @@ pub(super) async fn display_tree_with_segment_sizes( let segment_source = segment_source.clone(); async move { let buffer = segment_source.request(segment_id).await?; - let parts = ArrayParts::try_from(buffer)?; + let parts = SerializedArray::try_from(buffer)?; VortexResult::Ok((segment_id, parts.buffer_lengths())) } }); @@ -80,7 +80,7 @@ fn format_flat_layout_buffers( // First, try to get buffer info from inline array_tree if let Some(array_tree) = flat_layout.array_tree() - && let Ok(parts) = ArrayParts::from_array_tree(array_tree.as_ref().to_vec()) + && let Ok(parts) = SerializedArray::from_array_tree(array_tree.as_ref().to_vec()) { return format_buffer_sizes(&parts.buffer_lengths(), *segment_id); } @@ -225,7 +225,7 @@ mod tests { use vortex_array::dtype::Nullability::NonNullable; use vortex_array::dtype::PType; use vortex_array::dtype::StructFields; - use vortex_array::serde::ArrayParts; + use vortex_array::serde::SerializedArray; use vortex_array::validity::Validity; use vortex_buffer::BitBufferMut; use vortex_buffer::buffer; @@ -425,7 +425,7 @@ vortex.chunked, dtype: i32, children: 2, rows: 10 .array_tree() .expect("array_tree should be populated when FLAT_LAYOUT_INLINE_ARRAY_NODE is set"); - let parts = ArrayParts::from_array_tree(array_tree.as_ref().to_vec()) + let parts = SerializedArray::from_array_tree(array_tree.as_ref().to_vec()) .expect("should parse array_tree"); assert_eq!(parts.buffer_lengths(), vec![20]); // 5 i32 values = 20 bytes diff --git a/vortex-layout/src/layouts/flat/reader.rs b/vortex-layout/src/layouts/flat/reader.rs index 779cf12b3eb..7922e1c1560 100644 --- a/vortex-layout/src/layouts/flat/reader.rs +++ b/vortex-layout/src/layouts/flat/reader.rs @@ -14,7 +14,7 @@ use vortex_array::VortexSessionExecute; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; use vortex_array::expr::Expression; -use vortex_array::serde::ArrayParts; +use vortex_array::serde::SerializedArray; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_mask::Mask; @@ -72,10 +72,10 @@ impl FlatReader { let segment = segment_fut.await?; let parts = if let Some(array_tree) = array_tree { // Use the pre-stored flatbuffer from layout metadata combined with segment buffers. - ArrayParts::from_flatbuffer_and_segment(array_tree, segment)? + SerializedArray::from_flatbuffer_and_segment(array_tree, segment)? } else { // Parse the flatbuffer from the segment itself. - ArrayParts::try_from(segment)? + SerializedArray::try_from(segment)? }; parts .decode(&dtype, row_count, &ctx, &session) diff --git a/vortex-python/src/arrays/py/array.rs b/vortex-python/src/arrays/py/array.rs index 9163ae0c37d..d60c9469e59 100644 --- a/vortex-python/src/arrays/py/array.rs +++ b/vortex-python/src/arrays/py/array.rs @@ -8,6 +8,10 @@ use pyo3::FromPyObject; use pyo3::Py; use pyo3::PyAny; use pyo3::prelude::*; +use vortex::array::Array; +use vortex::array::ArrayParts; +use vortex::array::ArrayRef; +use vortex::array::IntoArray; use vortex::array::stats::ArrayStats; use vortex::dtype::DType; @@ -55,3 +59,15 @@ impl<'py> IntoPyObject<'py> for PythonArray { Ok(self.object.bind(py).to_owned()) } } + +impl IntoArray for PythonArray { + fn into_array(self) -> ArrayRef { + let vtable = self.vtable.clone(); + let dtype = self.dtype.clone(); + let len = self.len; + let stats = self.stats.clone(); + Array::try_from_parts(ArrayParts::new(vtable, dtype, len, self).with_stats(stats)) + .expect("PythonArray metadata extracted from PyPythonArray must be valid") + .into_array() + } +} diff --git a/vortex-python/src/arrays/py/vtable.rs b/vortex-python/src/arrays/py/vtable.rs index 3f323295020..dbbc767f253 100644 --- a/vortex-python/src/arrays/py/vtable.rs +++ b/vortex-python/src/arrays/py/vtable.rs @@ -4,9 +4,6 @@ use std::hash::Hash; use std::sync::Arc; -use pyo3::intern; -use pyo3::prelude::*; -use pyo3::types::PyBytes; use vortex::array::Array; use vortex::array::ArrayId; use vortex::array::ArrayRef; @@ -15,19 +12,15 @@ use vortex::array::ExecutionCtx; use vortex::array::ExecutionResult; use vortex::array::OperationsVTable; use vortex::array::Precision; -use vortex::array::RawMetadata; -use vortex::array::SerializeMetadata; use vortex::array::VTable; use vortex::array::ValidityVTable; use vortex::array::buffer::BufferHandle; -use vortex::array::serde::ArrayChildren; -use vortex::array::stats::ArrayStats; use vortex::array::validity::Validity; use vortex::array::vtable; use vortex::dtype::DType; use vortex::error::VortexResult; +use vortex::error::vortex_bail; use vortex::error::vortex_ensure; -use vortex::error::vortex_err; use vortex::error::vortex_panic; use vortex::scalar::Scalar; use vortex::session::VortexSession; @@ -45,28 +38,18 @@ pub struct PythonVTable { impl VTable for PythonVTable { type ArrayData = PythonArray; - type Metadata = RawMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(array: &Self::ArrayData) -> &Self { - &array.vtable - } - fn id(&self) -> ArrayId { self.id.clone() } - fn len(array: &PythonArray) -> usize { - array.len - } - - fn dtype(array: &PythonArray) -> &DType { - &array.dtype - } - - fn stats(array: &PythonArray) -> &ArrayStats { - &array.stats + fn validate(&self, data: &PythonArray, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!(data.vtable.id == self.id, "PythonArray vtable id mismatch"); + vortex_ensure!(&data.dtype == dtype, "PythonArray dtype mismatch"); + vortex_ensure!(data.len == len, "PythonArray len mismatch"); + Ok(()) } fn array_hash(array: &PythonArray, state: &mut H, _precision: Precision) { @@ -101,51 +84,21 @@ impl VTable for PythonVTable { vortex_panic!("PythonArray child_name index {idx} out of bounds") } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Python::attach(|py| { - let obj = array.object.bind(py); - if !obj - .hasattr(intern!(py, "metadata")) - .map_err(|e| vortex_err!("{}", e))? - { - // The class does not have a metadata attribute so does not support serialization. - return Ok(RawMetadata(vec![])); - } - - let bytes = obj - .call_method(intern!(py, "__vx_metadata__"), (), None) - .map_err(|e| vortex_err!("{}", e))? - .cast::() - .map_err(|_| vortex_err!("Expected array metadata to be Python bytes"))? - .as_bytes() - .to_vec(); - - Ok(RawMetadata(bytes)) - }) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(None) } fn deserialize( - bytes: &[u8], + &self, _dtype: &DType, _len: usize, + bytes: &[u8], _buffers: &[BufferHandle], + _children: &dyn vortex::array::serde::ArrayChildren, _session: &VortexSession, - ) -> VortexResult { - Ok(RawMetadata(bytes.to_vec())) - } - - fn build( - _dtype: &DType, - _len: usize, - _metadata: &Self::Metadata, - _buffers: &[BufferHandle], - _children: &dyn ArrayChildren, ) -> VortexResult { - todo!() + _ = bytes; + vortex_bail!("PythonArray deserialization is not supported"); } fn slots(_array: ArrayView<'_, Self>) -> &[Option] { diff --git a/vortex-python/src/serde/parts.rs b/vortex-python/src/serde/parts.rs index 2decc0a926a..5f112268e71 100644 --- a/vortex-python/src/serde/parts.rs +++ b/vortex-python/src/serde/parts.rs @@ -11,7 +11,7 @@ use pyo3::intern; use pyo3::prelude::PyAnyMethods; use pyo3::pyclass; use pyo3::pymethods; -use vortex::array::serde::ArrayParts; +use vortex::array::serde::SerializedArray; use vortex::buffer::ByteBuffer; use crate::SESSION; @@ -24,18 +24,18 @@ use crate::serde::context::PyReadContext; /// /// It can be decoded into a full array using the `decode` method. #[pyclass(name = "ArrayParts", module = "vortex", frozen)] -pub(crate) struct PyArrayParts(ArrayParts); +pub(crate) struct PyArrayParts(SerializedArray); impl Deref for PyArrayParts { - type Target = ArrayParts; + type Target = SerializedArray; fn deref(&self) -> &Self::Target { &self.0 } } -impl From for PyArrayParts { - fn from(parts: ArrayParts) -> Self { +impl From for PyArrayParts { + fn from(parts: SerializedArray) -> Self { Self(parts) } } @@ -47,7 +47,7 @@ impl PyArrayParts { fn parse(data: &[u8]) -> PyVortexResult { // TODO(ngates): create a buffer from a slice of bytes? let buffer = ByteBuffer::copy_from(data); - Ok(PyArrayParts(ArrayParts::try_from(buffer)?)) + Ok(PyArrayParts(SerializedArray::try_from(buffer)?)) } /// Decode the array parts into a full array. diff --git a/vortex-tui/src/browse/app.rs b/vortex-tui/src/browse/app.rs index a6292d071a8..340f19fbed6 100644 --- a/vortex-tui/src/browse/app.rs +++ b/vortex-tui/src/browse/app.rs @@ -352,7 +352,7 @@ impl AppState { #[cfg(feature = "native")] pub(crate) async fn load_flat_data(&mut self) { use vortex::array::MaskFuture; - use vortex::array::serde::ArrayParts; + use vortex::array::serde::SerializedArray; use vortex::expr::root; let layout = &self.cursor.layout().clone(); @@ -384,7 +384,7 @@ impl AppState { .await .vortex_expect("Failed to read segment"); self.cached_flatbuffer_size = Some( - ArrayParts::try_from(segment) + SerializedArray::try_from(segment) .vortex_expect("Failed to parse segment") .metadata() .len(), diff --git a/vortex-tui/src/wasm.rs b/vortex-tui/src/wasm.rs index f8acae67c61..8b643161fe2 100644 --- a/vortex-tui/src/wasm.rs +++ b/vortex-tui/src/wasm.rs @@ -14,7 +14,7 @@ use ratzilla::CanvasBackend; use ratzilla::WebRenderer; use ratzilla::ratatui::Terminal; use vortex::array::MaskFuture; -use vortex::array::serde::ArrayParts; +use vortex::array::serde::SerializedArray; use vortex::error::VortexExpect; use vortex::expr::root; use vortex::layout::layouts::flat::Flat; @@ -102,7 +102,7 @@ async fn load_flatbuffer_size( .request(segment_id) .await .vortex_expect("Failed to read segment"); - ArrayParts::try_from(segment) + SerializedArray::try_from(segment) .vortex_expect("Failed to parse segment") .metadata() .len() diff --git a/vortex-web/crate/src/wasm.rs b/vortex-web/crate/src/wasm.rs index bd4782079ff..86459f34fa4 100644 --- a/vortex-web/crate/src/wasm.rs +++ b/vortex-web/crate/src/wasm.rs @@ -25,7 +25,7 @@ use vortex::array::VortexSessionExecute; use vortex::array::arrow::ArrowArrayExecutor; use vortex::array::buffer::BufferHandle; use vortex::array::dtype::DType; -use vortex::array::serde::ArrayParts; +use vortex::array::serde::SerializedArray; use vortex::array::stream::ArrayStream; use vortex::buffer::Alignment; use vortex::buffer::ByteBufferMut; @@ -361,7 +361,8 @@ impl VortexFileHandle { .await .map_err(|e| JsValue::from_str(&e.to_string()))?; - let parts = ArrayParts::try_from(buf).map_err(|e| JsValue::from_str(&e.to_string()))?; + let parts = + SerializedArray::try_from(buf).map_err(|e| JsValue::from_str(&e.to_string()))?; let array = parts .decode(&dtype, row_count, ctx, &self.session) @@ -410,7 +411,8 @@ impl VortexFileHandle { .await .map_err(|e| JsValue::from_str(&e.to_string()))?; - let parts = ArrayParts::try_from(buf).map_err(|e| JsValue::from_str(&e.to_string()))?; + let parts = + SerializedArray::try_from(buf).map_err(|e| JsValue::from_str(&e.to_string()))?; let root_array = parts .decode(&dtype, row_count, ctx, &self.session) @@ -478,7 +480,8 @@ impl VortexFileHandle { .await .map_err(|e| JsValue::from_str(&e.to_string()))?; - let parts = ArrayParts::try_from(buf).map_err(|e| JsValue::from_str(&e.to_string()))?; + let parts = + SerializedArray::try_from(buf).map_err(|e| JsValue::from_str(&e.to_string()))?; let root_array = parts .decode(&dtype, row_count, ctx, &self.session) @@ -583,7 +586,7 @@ fn build_layout_tree( let array_encoding_tree = layout.as_opt::().and_then(|flat| { let tree_buf = flat.array_tree()?; let ctx = flat.array_ctx(); - let parts = ArrayParts::from_array_tree(tree_buf.as_ref().to_vec()).ok()?; + let parts = SerializedArray::from_array_tree(tree_buf.as_ref().to_vec()).ok()?; Some(build_array_encoding_tree(&parts, ctx)) }); @@ -612,7 +615,7 @@ fn sum_metadata_bytes(layout: &LayoutRef) -> VortexResult { /// Recursively build the array encoding tree from `ArrayParts` (used for inline trees /// where we don't have a fully decoded array). -fn build_array_encoding_tree(parts: &ArrayParts, ctx: &ReadContext) -> ArrayEncodingNodeJson { +fn build_array_encoding_tree(parts: &SerializedArray, ctx: &ReadContext) -> ArrayEncodingNodeJson { let encoding = ctx .resolve(parts.encoding_id()) .map(|id| id.to_string()) From 8b80dd8ea15b7d8d47bdac5779998e1fcb6eed92 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Fri, 3 Apr 2026 07:24:15 -0400 Subject: [PATCH 04/15] Advanced frame parsing Signed-off-by: Nicholas Gates --- encodings/alp/public-api.lock | 74 +- encodings/alp/src/alp/array.rs | 11 +- encodings/alp/src/alp/compress.rs | 4 +- encodings/alp/src/alp/compute/cast.rs | 3 +- encodings/bytebool/public-api.lock | 34 +- encodings/bytebool/src/array.rs | 26 +- encodings/bytebool/src/compute.rs | 16 +- encodings/datetime-parts/public-api.lock | 32 +- encodings/datetime-parts/src/canonical.rs | 16 +- encodings/datetime-parts/src/compute/cast.rs | 16 +- .../datetime-parts/src/compute/compare.rs | 9 +- .../datetime-parts/src/compute/filter.rs | 6 +- encodings/datetime-parts/src/compute/mod.rs | 38 +- encodings/datetime-parts/src/compute/rules.rs | 7 +- encodings/datetime-parts/src/compute/take.rs | 20 +- encodings/decimal-byte-parts/public-api.lock | 34 +- encodings/fastlanes/public-api.lock | 174 +- .../src/bitpacking/array/bitpack_compress.rs | 11 +- .../bitpacking/array/bitpack_decompress.rs | 76 +- .../fastlanes/src/bitpacking/array/mod.rs | 6 +- .../src/bitpacking/array/unpack_iter.rs | 3 +- .../src/bitpacking/compute/filter.rs | 33 +- .../fastlanes/src/bitpacking/compute/slice.rs | 4 +- .../fastlanes/src/bitpacking/compute/take.rs | 10 +- .../src/bitpacking/vtable/operations.rs | 57 +- .../src/delta/array/delta_compress.rs | 11 +- encodings/fastlanes/src/delta/array/mod.rs | 13 - encodings/fastlanes/src/delta/compute/cast.rs | 12 +- encodings/fastlanes/src/delta/vtable/slice.rs | 3 +- .../fastlanes/src/for/array/for_compress.rs | 3 +- encodings/fastlanes/src/for/array/mod.rs | 12 +- encodings/fastlanes/src/for/compute/cast.rs | 4 +- .../fastlanes/src/for/compute/compare.rs | 25 +- encodings/fastlanes/src/for/compute/mod.rs | 18 +- encodings/fastlanes/src/for/vtable/slice.rs | 7 +- encodings/fastlanes/src/rle/array/mod.rs | 75 +- .../fastlanes/src/rle/array/rle_compress.rs | 2 +- encodings/fastlanes/src/rle/compute/cast.rs | 5 +- .../fastlanes/src/rle/vtable/operations.rs | 31 +- encodings/fsst/public-api.lock | 32 +- encodings/pco/public-api.lock | 32 +- encodings/pco/src/compute/cast.rs | 5 +- encodings/pco/src/test.rs | 5 +- encodings/runend/public-api.lock | 32 +- encodings/runend/src/arrow.rs | 9 +- encodings/sequence/public-api.lock | 40 +- encodings/sparse/public-api.lock | 36 +- encodings/sparse/src/compute/filter.rs | 3 +- encodings/sparse/src/rules.rs | 4 +- encodings/sparse/src/slice.rs | 3 +- encodings/zigzag/public-api.lock | 28 +- encodings/zigzag/src/slice.rs | 4 +- encodings/zstd/public-api.lock | 42 +- encodings/zstd/src/slice.rs | 7 +- encodings/zstd/src/test.rs | 12 +- vortex-array/public-api.lock | 2860 +++-------------- vortex-array/src/array/typed.rs | 36 +- vortex-array/src/arrays/bool/array.rs | 4 +- vortex-array/src/arrays/chunked/array.rs | 20 +- vortex-array/src/arrays/chunked/tests.rs | 5 +- vortex-array/src/arrays/chunked/vtable/mod.rs | 37 +- vortex-array/src/arrays/constant/array.rs | 5 +- vortex-array/src/arrays/dict/vtable/mod.rs | 28 +- .../src/arrays/dict/vtable/validity.rs | 12 +- .../src/arrays/extension/compute/cast.rs | 2 +- .../src/arrays/extension/vtable/mod.rs | 3 +- .../src/arrays/fixed_size_list/array.rs | 4 +- .../src/arrays/fixed_size_list/vtable/mod.rs | 3 +- vortex-array/src/arrays/list/array.rs | 2 +- vortex-array/src/arrays/list/tests.rs | 20 +- vortex-array/src/arrays/list/vtable/mod.rs | 15 +- vortex-array/src/arrays/listview/array.rs | 1 - .../src/arrays/listview/vtable/mod.rs | 17 +- .../src/arrays/masked/compute/filter.rs | 4 +- .../src/arrays/masked/compute/take.rs | 4 +- vortex-array/src/arrays/masked/vtable/mod.rs | 5 +- vortex-array/src/arrays/patched/array.rs | 59 +- .../src/arrays/patched/compute/compare.rs | 26 +- .../src/arrays/patched/compute/filter.rs | 2 +- .../src/arrays/patched/compute/take.rs | 2 +- vortex-array/src/arrays/patched/vtable/mod.rs | 9 +- .../src/arrays/patched/vtable/slice.rs | 5 - .../src/arrays/scalar_fn/vtable/mod.rs | 5 +- vortex-array/src/arrays/struct_/array.rs | 102 +- .../src/arrays/struct_/compute/rules.rs | 2 +- .../src/arrays/struct_/compute/zip.rs | 2 +- vortex-array/src/arrays/struct_/vtable/mod.rs | 7 +- vortex-array/src/arrays/varbin/array.rs | 49 +- vortex-array/src/arrays/varbin/vtable/mod.rs | 19 +- vortex-array/src/arrays/varbinview/array.rs | 48 +- .../src/arrays/varbinview/vtable/mod.rs | 4 + vortex-array/src/arrays/variant/mod.rs | 4 +- vortex-array/src/arrays/variant/vtable/mod.rs | 3 +- vortex-array/src/arrow/executor/struct_.rs | 4 +- vortex-array/src/canonical.rs | 8 +- .../src/display/extractors/metadata.rs | 1 + vortex-array/src/serde.rs | 10 +- vortex-btrblocks/src/schemes/integer.rs | 5 +- vortex-ipc/public-api.lock | 2 +- vortex-python/python/vortex/__init__.py | 4 +- vortex-python/python/vortex/_lib/serde.pyi | 6 +- vortex-python/python/vortex/arrays.py | 10 +- vortex-python/python/vortex/serde.py | 4 +- vortex-python/src/serde/mod.rs | 4 +- vortex-python/src/serde/parts.rs | 22 +- vortex-python/test/test_pyarray.py | 2 +- 106 files changed, 1374 insertions(+), 3378 deletions(-) diff --git a/encodings/alp/public-api.lock b/encodings/alp/public-api.lock index 9f380eb7450..a130df6493b 100644 --- a/encodings/alp/public-api.lock +++ b/encodings/alp/public-api.lock @@ -12,7 +12,7 @@ impl vortex_alp::ALP pub fn vortex_alp::ALP::new(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option) -> vortex_alp::ALPArray -pub unsafe fn vortex_alp::ALP::new_unchecked(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option, dtype: vortex_array::dtype::DType) -> vortex_alp::ALPArray +pub unsafe fn vortex_alp::ALP::new_unchecked(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option) -> vortex_alp::ALPArray pub fn vortex_alp::ALP::try_new(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option) -> vortex_error::VortexResult @@ -28,8 +28,6 @@ impl vortex_array::array::vtable::VTable for vortex_alp::ALP pub type vortex_alp::ALP::ArrayData = vortex_alp::ALPData -pub type vortex_alp::ALP::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_alp::ALP::OperationsVTable = vortex_alp::ALP pub type vortex_alp::ALP::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -42,11 +40,7 @@ pub fn vortex_alp::ALP::buffer(_array: vortex_array::array::view::ArrayView<'_, pub fn vortex_alp::ALP::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_alp::ALP::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_alp::ALP::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_alp::ALP::dtype(array: &vortex_alp::ALPData) -> &vortex_array::dtype::DType +pub fn vortex_alp::ALP::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_alp::ALP::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -54,23 +48,17 @@ pub fn vortex_alp::ALP::execute_parent(array: vortex_array::array::view::ArrayVi pub fn vortex_alp::ALP::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_alp::ALP::len(array: &vortex_alp::ALPData) -> usize - -pub fn vortex_alp::ALP::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_alp::ALP::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_alp::ALP::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_alp::ALP::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_alp::ALP::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_alp::ALP::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_alp::ALP::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_alp::ALP::stats(array: &vortex_alp::ALPData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_alp::ALP::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_alp::ALP::validate(&self, data: &vortex_alp::ALPData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_alp::ALP::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -118,22 +106,14 @@ pub struct vortex_alp::ALPData impl vortex_alp::ALPData -pub fn vortex_alp::ALPData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_alp::ALPData::encoded(&self) -> &vortex_array::array::erased::ArrayRef pub fn vortex_alp::ALPData::exponents(&self) -> vortex_alp::Exponents -pub fn vortex_alp::ALPData::into_parts(self) -> (vortex_array::array::erased::ArrayRef, vortex_alp::Exponents, core::option::Option, vortex_array::dtype::DType) - -pub fn vortex_alp::ALPData::is_empty(&self) -> bool - -pub fn vortex_alp::ALPData::len(&self) -> usize +pub fn vortex_alp::ALPData::into_parts(self) -> (vortex_array::array::erased::ArrayRef, vortex_alp::Exponents, core::option::Option) pub fn vortex_alp::ALPData::patches(&self) -> core::option::Option -pub fn vortex_alp::ALPData::ptype(&self) -> vortex_array::dtype::ptype::PType - impl vortex_alp::ALPData pub fn vortex_alp::ALPData::new(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option) -> Self @@ -144,18 +124,10 @@ impl core::clone::Clone for vortex_alp::ALPData pub fn vortex_alp::ALPData::clone(&self) -> vortex_alp::ALPData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_alp::ALPData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_alp::ALPData pub fn vortex_alp::ALPData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_alp::ALPData - -pub fn vortex_alp::ALPData::into_array(self) -> vortex_array::array::erased::ArrayRef - pub struct vortex_alp::ALPMetadata impl core::clone::Clone for vortex_alp::ALPMetadata @@ -198,8 +170,6 @@ impl vortex_array::array::vtable::VTable for vortex_alp::ALPRD pub type vortex_alp::ALPRD::ArrayData = vortex_alp::ALPRDData -pub type vortex_alp::ALPRD::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_alp::ALPRD::OperationsVTable = vortex_alp::ALPRD pub type vortex_alp::ALPRD::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -212,11 +182,7 @@ pub fn vortex_alp::ALPRD::buffer(_array: vortex_array::array::view::ArrayView<'_ pub fn vortex_alp::ALPRD::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_alp::ALPRD::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_alp::ALPRD::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_alp::ALPRD::dtype(array: &vortex_alp::ALPRDData) -> &vortex_array::dtype::DType +pub fn vortex_alp::ALPRD::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_alp::ALPRD::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -224,23 +190,17 @@ pub fn vortex_alp::ALPRD::execute_parent(array: vortex_array::array::view::Array pub fn vortex_alp::ALPRD::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_alp::ALPRD::len(array: &vortex_alp::ALPRDData) -> usize - -pub fn vortex_alp::ALPRD::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_alp::ALPRD::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_alp::ALPRD::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_alp::ALPRD::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_alp::ALPRD::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_alp::ALPRD::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_alp::ALPRD::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_alp::ALPRD::stats(array: &vortex_alp::ALPRDData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_alp::ALPRD::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_alp::ALPRD::validate(&self, data: &vortex_alp::ALPRDData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_alp::ALPRD::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -274,8 +234,6 @@ pub fn vortex_alp::ALPRD::mask(array: vortex_array::array::view::ArrayView<'_, S pub struct vortex_alp::ALPRDArrayParts -pub vortex_alp::ALPRDArrayParts::dtype: vortex_array::dtype::DType - pub vortex_alp::ALPRDArrayParts::left_parts: vortex_array::array::erased::ArrayRef pub vortex_alp::ALPRDArrayParts::left_parts_dictionary: vortex_buffer::buffer::Buffer @@ -296,22 +254,14 @@ pub struct vortex_alp::ALPRDData impl vortex_alp::ALPRDData -pub fn vortex_alp::ALPRDData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_alp::ALPRDData::into_parts(self) -> vortex_alp::ALPRDArrayParts -pub fn vortex_alp::ALPRDData::is_empty(&self) -> bool - -pub fn vortex_alp::ALPRDData::is_f32(&self) -> bool - pub fn vortex_alp::ALPRDData::left_parts(&self) -> &vortex_array::array::erased::ArrayRef pub fn vortex_alp::ALPRDData::left_parts_dictionary(&self) -> &vortex_buffer::buffer::Buffer pub fn vortex_alp::ALPRDData::left_parts_patches(&self) -> core::option::Option -pub fn vortex_alp::ALPRDData::len(&self) -> usize - pub fn vortex_alp::ALPRDData::replace_left_parts_patches(&mut self, patches: core::option::Option) pub fn vortex_alp::ALPRDData::right_bit_width(&self) -> u8 @@ -324,18 +274,10 @@ impl core::clone::Clone for vortex_alp::ALPRDData pub fn vortex_alp::ALPRDData::clone(&self) -> vortex_alp::ALPRDData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_alp::ALPRDData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_alp::ALPRDData pub fn vortex_alp::ALPRDData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_alp::ALPRDData - -pub fn vortex_alp::ALPRDData::into_array(self) -> vortex_array::array::erased::ArrayRef - pub struct vortex_alp::ALPRDMetadata impl vortex_alp::ALPRDMetadata diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index 17d624f455a..322d67e6835 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -373,7 +373,7 @@ impl ALPData { /// # Examples /// /// ``` - /// # use vortex_alp::{ALPData, ALPArray, Exponents}; + /// # use vortex_alp::{ALP, ALPData, Exponents}; /// # use vortex_array::IntoArray; /// # use vortex_buffer::buffer; /// @@ -394,11 +394,11 @@ impl ALPData { /// assert!(result.is_err()); /// /// // Success! - /// let value = ALPArray::try_from_data(ALPData::try_new( + /// let value = ALP::try_new( /// buffer![0i32].into_array(), /// Exponents { e: 1, f: 1 }, /// None - /// ).unwrap()).unwrap(); + /// ).unwrap(); /// /// assert_eq!(value.scalar_at(0).unwrap(), 0f32.into()); /// ``` @@ -872,12 +872,11 @@ mod tests { .unwrap(); // Build a new ALPArray with the same encoded data but patches without chunk_offsets. - let alp_without_chunk_offsets = ALPArray::try_from_data(ALPData::new( + let alp_without_chunk_offsets = ALP::new( normally_encoded.encoded().clone(), normally_encoded.exponents(), Some(patches_without_chunk_offsets), - )) - .vortex_expect("ALPData is always valid"); + ); // The legacy decompress_into_array path should work correctly. let result_legacy = decompress_into_array( diff --git a/encodings/alp/src/alp/compress.rs b/encodings/alp/src/alp/compress.rs index 668799f8984..a55baf02698 100644 --- a/encodings/alp/src/alp/compress.rs +++ b/encodings/alp/src/alp/compress.rs @@ -47,9 +47,7 @@ pub fn alp_encode(parray: &PrimitiveArray, exponents: Option) -> Vort }; // SAFETY: alp_encode_components_typed must return well-formed components - unsafe { - Ok(ALP::new_unchecked(encoded, exponents, patches)) - } + unsafe { Ok(ALP::new_unchecked(encoded, exponents, patches)) } } #[expect( diff --git a/encodings/alp/src/alp/compute/cast.rs b/encodings/alp/src/alp/compute/cast.rs index d07b2a7d128..bd8a8b38381 100644 --- a/encodings/alp/src/alp/compute/cast.rs +++ b/encodings/alp/src/alp/compute/cast.rs @@ -45,8 +45,7 @@ impl CastReduce for ALP { // SAFETY: casting nullability doesn't alter the invariants unsafe { Ok(Some( - ALP::new_unchecked(new_encoded, array.exponents(), new_patches) - .into_array(), + ALP::new_unchecked(new_encoded, array.exponents(), new_patches).into_array(), )) } } else { diff --git a/encodings/bytebool/public-api.lock b/encodings/bytebool/public-api.lock index 5f05421a6d6..74eed2ce798 100644 --- a/encodings/bytebool/public-api.lock +++ b/encodings/bytebool/public-api.lock @@ -6,8 +6,12 @@ impl vortex_bytebool::ByteBool pub const vortex_bytebool::ByteBool::ID: vortex_array::array::ArrayId +pub fn vortex_bytebool::ByteBool::from_option_vec(data: alloc::vec::Vec>) -> vortex_bytebool::ByteBoolArray + pub fn vortex_bytebool::ByteBool::from_vec>(data: alloc::vec::Vec, validity: V) -> vortex_bytebool::ByteBoolArray +pub fn vortex_bytebool::ByteBool::new(buffer: vortex_array::buffer::BufferHandle, validity: vortex_array::validity::Validity) -> vortex_bytebool::ByteBoolArray + impl core::clone::Clone for vortex_bytebool::ByteBool pub fn vortex_bytebool::ByteBool::clone(&self) -> vortex_bytebool::ByteBool @@ -20,8 +24,6 @@ impl vortex_array::array::vtable::VTable for vortex_bytebool::ByteBool pub type vortex_bytebool::ByteBool::ArrayData = vortex_bytebool::ByteBoolData -pub type vortex_bytebool::ByteBool::Metadata = vortex_array::metadata::EmptyMetadata - pub type vortex_bytebool::ByteBool::OperationsVTable = vortex_bytebool::ByteBool pub type vortex_bytebool::ByteBool::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromValidityHelper @@ -34,11 +36,7 @@ pub fn vortex_bytebool::ByteBool::buffer(array: vortex_array::array::view::Array pub fn vortex_bytebool::ByteBool::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_bytebool::ByteBool::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_bytebool::ByteBool::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_bytebool::ByteBool::dtype(array: &vortex_bytebool::ByteBoolData) -> &vortex_array::dtype::DType +pub fn vortex_bytebool::ByteBool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_bytebool::ByteBool::execute(array: vortex_array::array::typed::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -46,23 +44,17 @@ pub fn vortex_bytebool::ByteBool::execute_parent(array: vortex_array::array::vie pub fn vortex_bytebool::ByteBool::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_bytebool::ByteBool::len(array: &vortex_bytebool::ByteBoolData) -> usize - -pub fn vortex_bytebool::ByteBool::metadata(_array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_bytebool::ByteBool::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_bytebool::ByteBool::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_bytebool::ByteBool::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_bytebool::ByteBool::serialize(_array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_bytebool::ByteBool::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_bytebool::ByteBool::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_bytebool::ByteBool::stats(array: &vortex_bytebool::ByteBoolData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_bytebool::ByteBool::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_bytebool::ByteBool::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_bytebool::ByteBool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -94,8 +86,6 @@ pub fn vortex_bytebool::ByteBoolData::as_slice(&self) -> &[bool] pub fn vortex_bytebool::ByteBoolData::buffer(&self) -> &vortex_array::buffer::BufferHandle -pub fn vortex_bytebool::ByteBoolData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_bytebool::ByteBoolData::from_vec>(data: alloc::vec::Vec, validity: V) -> Self pub fn vortex_bytebool::ByteBoolData::is_empty(&self) -> bool @@ -104,6 +94,8 @@ pub fn vortex_bytebool::ByteBoolData::len(&self) -> usize pub fn vortex_bytebool::ByteBoolData::new(buffer: vortex_array::buffer::BufferHandle, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_bytebool::ByteBoolData::validate(buffer: &vortex_array::buffer::BufferHandle, validity: &vortex_array::validity::Validity, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> + pub fn vortex_bytebool::ByteBoolData::validity_mask(&self) -> vortex_mask::Mask impl core::clone::Clone for vortex_bytebool::ByteBoolData @@ -118,18 +110,10 @@ impl core::convert::From>> for vortex pub fn vortex_bytebool::ByteBoolData::from(value: alloc::vec::Vec>) -> Self -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_bytebool::ByteBoolData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_bytebool::ByteBoolData pub fn vortex_bytebool::ByteBoolData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_bytebool::ByteBoolData - -pub fn vortex_bytebool::ByteBoolData::into_array(self) -> vortex_array::array::erased::ArrayRef - impl vortex_array::array::vtable::validity::ValidityHelper for vortex_bytebool::ByteBoolData pub fn vortex_bytebool::ByteBoolData::validity(&self) -> &vortex_array::validity::Validity diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index aadb8405556..ebde50f0a90 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -197,6 +197,15 @@ impl ByteBool { Array::try_from_parts(ArrayParts::new(ByteBool, dtype, len, data)) .vortex_expect("ByteBoolData is always valid") } + + /// Construct a [`ByteBoolArray`] from optional bools. + pub fn from_option_vec(data: Vec>) -> ByteBoolArray { + let data = ByteBoolData::from(data); + let dtype = DType::Bool(data.validity.nullability()); + let len = data.len(); + Array::try_from_parts(ArrayParts::new(ByteBool, dtype, len, data)) + .vortex_expect("ByteBoolData is always valid") + } } impl ByteBoolData { @@ -334,8 +343,7 @@ mod tests { let v = vec![true, false]; let v_len = v.len(); - let arr = ByteBoolArray::try_from_data(ByteBoolData::from(v)) - .vortex_expect("ByteBoolData is always valid"); + let arr = ByteBool::from_vec(v, Validity::AllValid); assert_eq!(v_len, arr.len()); for idx in 0..arr.len() { @@ -343,8 +351,7 @@ mod tests { } let v = vec![Some(true), None, Some(false)]; - let arr = ByteBoolArray::try_from_data(ByteBoolData::from(v)) - .vortex_expect("ByteBoolData is always valid"); + let arr = ByteBool::from_option_vec(v); assert!(arr.is_valid(0).unwrap()); assert!(!arr.is_valid(1).unwrap()); assert!(arr.is_valid(2).unwrap()); @@ -353,8 +360,7 @@ mod tests { let v: Vec> = vec![None, None]; let v_len = v.len(); - let arr = ByteBoolArray::try_from_data(ByteBoolData::from(v)) - .vortex_expect("ByteBoolData is always valid"); + let arr = ByteBool::from_option_vec(v); assert_eq!(v_len, arr.len()); for idx in 0..arr.len() { @@ -365,13 +371,7 @@ mod tests { #[test] fn test_nullable_bytebool_serde_roundtrip() { - let array = ByteBoolArray::try_from_data(ByteBoolData::from(vec![ - Some(true), - None, - Some(false), - None, - ])) - .unwrap(); + let array = ByteBool::from_option_vec(vec![Some(true), None, Some(false), None]); let dtype = array.dtype().clone(); let len = array.len(); let session = VortexSession::empty().with::(); diff --git a/encodings/bytebool/src/compute.rs b/encodings/bytebool/src/compute.rs index 609a83f799a..d929ed7b2c8 100644 --- a/encodings/bytebool/src/compute.rs +++ b/encodings/bytebool/src/compute.rs @@ -30,7 +30,9 @@ impl CastReduce for ByteBool { .clone() .cast_nullability(dtype.nullability(), array.len())?; - return Ok(Some(ByteBool::new(array.buffer().clone(), new_validity).into_array())); + return Ok(Some( + ByteBool::new(array.buffer().clone(), new_validity).into_array(), + )); } // For other casts, decode to canonical and let BoolArray handle it @@ -82,6 +84,8 @@ impl TakeExecute for ByteBool { #[cfg(test)] mod tests { + use super::*; + use crate::ByteBoolArray; use rstest::rstest; use vortex_array::assert_arrays_eq; use vortex_array::builtins::ArrayBuiltins; @@ -93,19 +97,13 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::scalar_fn::fns::operators::Operator; - use vortex_error::VortexExpect; - - use super::*; - use crate::ByteBoolArray; fn bb(v: Vec) -> ByteBoolArray { - ByteBoolArray::try_from_data(ByteBoolData::from(v)) - .vortex_expect("ByteBoolData is always valid") + ByteBool::from_vec(v, Validity::AllValid) } fn bb_opt(v: Vec>) -> ByteBoolArray { - ByteBoolArray::try_from_data(ByteBoolData::from(v)) - .vortex_expect("ByteBoolData is always valid") + ByteBool::from_option_vec(v) } #[test] diff --git a/encodings/datetime-parts/public-api.lock b/encodings/datetime-parts/public-api.lock index 0bb4af9c59b..d10034f6527 100644 --- a/encodings/datetime-parts/public-api.lock +++ b/encodings/datetime-parts/public-api.lock @@ -22,8 +22,6 @@ impl vortex_array::array::vtable::VTable for vortex_datetime_parts::DateTimePart pub type vortex_datetime_parts::DateTimeParts::ArrayData = vortex_datetime_parts::DateTimePartsData -pub type vortex_datetime_parts::DateTimeParts::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_datetime_parts::DateTimeParts::OperationsVTable = vortex_datetime_parts::DateTimeParts pub type vortex_datetime_parts::DateTimeParts::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -36,11 +34,7 @@ pub fn vortex_datetime_parts::DateTimeParts::buffer(_array: vortex_array::array: pub fn vortex_datetime_parts::DateTimeParts::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_datetime_parts::DateTimeParts::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_datetime_parts::DateTimeParts::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_datetime_parts::DateTimeParts::dtype(array: &vortex_datetime_parts::DateTimePartsData) -> &vortex_array::dtype::DType +pub fn vortex_datetime_parts::DateTimeParts::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_datetime_parts::DateTimeParts::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -48,23 +42,17 @@ pub fn vortex_datetime_parts::DateTimeParts::execute_parent(array: vortex_array: pub fn vortex_datetime_parts::DateTimeParts::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_datetime_parts::DateTimeParts::len(array: &vortex_datetime_parts::DateTimePartsData) -> usize - -pub fn vortex_datetime_parts::DateTimeParts::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_datetime_parts::DateTimeParts::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_datetime_parts::DateTimeParts::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_datetime_parts::DateTimeParts::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_datetime_parts::DateTimeParts::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_datetime_parts::DateTimeParts::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_datetime_parts::DateTimeParts::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_datetime_parts::DateTimeParts::stats(array: &vortex_datetime_parts::DateTimePartsData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_datetime_parts::DateTimeParts::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_datetime_parts::DateTimeParts::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_datetime_parts::DateTimeParts::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -124,10 +112,6 @@ impl vortex_datetime_parts::DateTimePartsData pub fn vortex_datetime_parts::DateTimePartsData::days(&self) -> &vortex_array::array::erased::ArrayRef -pub fn vortex_datetime_parts::DateTimePartsData::dtype(&self) -> &vortex_array::dtype::DType - -pub fn vortex_datetime_parts::DateTimePartsData::into_parts(self) -> vortex_datetime_parts::DateTimePartsArrayParts - pub fn vortex_datetime_parts::DateTimePartsData::is_empty(&self) -> bool pub fn vortex_datetime_parts::DateTimePartsData::len(&self) -> usize @@ -138,14 +122,12 @@ pub fn vortex_datetime_parts::DateTimePartsData::subseconds(&self) -> &vortex_ar pub fn vortex_datetime_parts::DateTimePartsData::try_new(dtype: vortex_array::dtype::DType, days: vortex_array::array::erased::ArrayRef, seconds: vortex_array::array::erased::ArrayRef, subseconds: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult +pub fn vortex_datetime_parts::DateTimePartsData::validate(dtype: &vortex_array::dtype::DType, days: &vortex_array::array::erased::ArrayRef, seconds: &vortex_array::array::erased::ArrayRef, subseconds: &vortex_array::array::erased::ArrayRef, len: usize) -> vortex_error::VortexResult<()> + impl core::clone::Clone for vortex_datetime_parts::DateTimePartsData pub fn vortex_datetime_parts::DateTimePartsData::clone(&self) -> vortex_datetime_parts::DateTimePartsData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_datetime_parts::DateTimePartsData) -> vortex_array::array::erased::ArrayRef - impl core::convert::TryFrom for vortex_datetime_parts::DateTimePartsData pub type vortex_datetime_parts::DateTimePartsData::Error = vortex_error::VortexError @@ -156,10 +138,6 @@ impl core::fmt::Debug for vortex_datetime_parts::DateTimePartsData pub fn vortex_datetime_parts::DateTimePartsData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_datetime_parts::DateTimePartsData - -pub fn vortex_datetime_parts::DateTimePartsData::into_array(self) -> vortex_array::array::erased::ArrayRef - #[repr(C)] pub struct vortex_datetime_parts::DateTimePartsMetadata pub vortex_datetime_parts::DateTimePartsMetadata::days_ptype: i32 diff --git a/encodings/datetime-parts/src/canonical.rs b/encodings/datetime-parts/src/canonical.rs index 99337f898a5..daecaecfd3f 100644 --- a/encodings/datetime-parts/src/canonical.rs +++ b/encodings/datetime-parts/src/canonical.rs @@ -119,8 +119,7 @@ mod test { use vortex_error::VortexResult; use vortex_session::VortexSession; - use crate::DateTimePartsArray; - use crate::DateTimePartsData; + use crate::DateTimeParts; use crate::canonical::decode_to_temporal; #[rstest] @@ -140,14 +139,11 @@ mod test { ], validity.clone(), ); - let date_times = DateTimePartsArray::try_from_data( - DateTimePartsData::try_from(TemporalArray::new_timestamp( - milliseconds.clone().into_array(), - TimeUnit::Milliseconds, - Some("UTC".into()), - )) - .unwrap(), - )?; + let date_times = DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( + milliseconds.clone().into_array(), + TimeUnit::Milliseconds, + Some("UTC".into()), + ))?; let mut ctx = ExecutionCtx::new(VortexSession::empty()); diff --git a/encodings/datetime-parts/src/compute/cast.rs b/encodings/datetime-parts/src/compute/cast.rs index cc60e253aaf..ca846bbd424 100644 --- a/encodings/datetime-parts/src/compute/cast.rs +++ b/encodings/datetime-parts/src/compute/cast.rs @@ -44,11 +44,11 @@ mod tests { use vortex_array::validity::Validity; use vortex_buffer::buffer; + use crate::DateTimeParts; use crate::DateTimePartsArray; - use crate::DateTimePartsData; fn date_time_array(validity: Validity) -> ArrayRef { - DateTimePartsData::try_from(TemporalArray::new_timestamp( + DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( PrimitiveArray::new( buffer![ 86_400i64, // element with only day component @@ -103,7 +103,7 @@ mod tests { } #[rstest] - #[case(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( buffer![ 0i64, 86_400_000, // 1 day in ms @@ -113,8 +113,8 @@ mod tests { ].into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap()).unwrap())] - #[case(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + )).unwrap())] + #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( PrimitiveArray::from_option_iter([ Some(0i64), None, @@ -124,12 +124,12 @@ mod tests { ]).into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap()).unwrap())] - #[case(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + )).unwrap())] + #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( buffer![86_400_000_000_000i64].into_array(), // 1 day in ns TimeUnit::Nanoseconds, Some("UTC".into()) - )).unwrap()).unwrap())] + )).unwrap())] fn test_cast_datetime_parts_conformance(#[case] array: DateTimePartsArray) { use vortex_array::compute::conformance::cast::test_cast_conformance; test_cast_conformance(&array.into_array()); diff --git a/encodings/datetime-parts/src/compute/compare.rs b/encodings/datetime-parts/src/compute/compare.rs index 0c92e6a5168..76250bd320e 100644 --- a/encodings/datetime-parts/src/compute/compare.rs +++ b/encodings/datetime-parts/src/compute/compare.rs @@ -205,13 +205,14 @@ mod test { use vortex_buffer::buffer; use super::*; - use crate::DateTimePartsData; + use crate::DateTimeParts; + use crate::DateTimePartsArray; fn dtp_array_from_timestamp( value: T, validity: Validity, - ) -> DateTimePartsData { - DateTimePartsData::try_from(TemporalArray::new_timestamp( + ) -> DateTimePartsArray { + DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( PrimitiveArray::new(buffer![value], validity).into_array(), TimeUnit::Seconds, Some("UTC".into()), @@ -312,7 +313,7 @@ mod test { Some("UTC".into()), ); - let lhs = DateTimePartsData::try_new( + let lhs = DateTimeParts::try_new( DType::Extension(temporal_array.ext_dtype()), PrimitiveArray::new(buffer![0i32], lhs_validity).into_array(), PrimitiveArray::new(buffer![0u32], Validity::NonNullable).into_array(), diff --git a/encodings/datetime-parts/src/compute/filter.rs b/encodings/datetime-parts/src/compute/filter.rs index 0320c5c1917..da825d8a95e 100644 --- a/encodings/datetime-parts/src/compute/filter.rs +++ b/encodings/datetime-parts/src/compute/filter.rs @@ -32,7 +32,7 @@ mod test { use vortex_array::extension::datetime::TimeUnit; use vortex_buffer::buffer; - use crate::DateTimePartsData; + use crate::DateTimeParts; #[test] fn test_filter_datetime_parts() { @@ -49,7 +49,7 @@ mod test { let temporal = TemporalArray::new_timestamp(timestamps, TimeUnit::Milliseconds, Some("UTC".into())); - let array = DateTimePartsData::try_from(temporal).unwrap(); + let array = DateTimeParts::try_from_temporal(temporal).unwrap(); test_filter_conformance(&array.into_array()); // Test with nullable values @@ -65,7 +65,7 @@ mod test { let temporal = TemporalArray::new_timestamp(timestamps, TimeUnit::Milliseconds, Some("UTC".into())); - let array = DateTimePartsData::try_from(temporal).unwrap(); + let array = DateTimeParts::try_from_temporal(temporal).unwrap(); test_filter_conformance(&array.into_array()); } } diff --git a/encodings/datetime-parts/src/compute/mod.rs b/encodings/datetime-parts/src/compute/mod.rs index 4ca8398fd6b..53717a52b0c 100644 --- a/encodings/datetime-parts/src/compute/mod.rs +++ b/encodings/datetime-parts/src/compute/mod.rs @@ -21,55 +21,59 @@ mod tests { use vortex_array::extension::datetime::TimeUnit; use vortex_buffer::buffer; + use crate::DateTimeParts; use crate::DateTimePartsArray; - use crate::DateTimePartsData; + + fn dtp_from_temporal(temporal: TemporalArray) -> DateTimePartsArray { + DateTimeParts::try_from_temporal(temporal).unwrap() + } #[rstest] // Basic datetime arrays - #[case::datetime_seconds(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + #[case::datetime_seconds(dtp_from_temporal(TemporalArray::new_timestamp( buffer![0i64, 86400, 172800, 259200, 345600].into_array(), TimeUnit::Seconds, Some("UTC".into()), - )).unwrap()).unwrap())] - #[case::datetime_millis(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + )))] + #[case::datetime_millis(dtp_from_temporal(TemporalArray::new_timestamp( buffer![0i64, 86400000, 172800000].into_array(), TimeUnit::Milliseconds, Some("UTC".into()), - )).unwrap()).unwrap())] - #[case::datetime_micros(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + )))] + #[case::datetime_micros(dtp_from_temporal(TemporalArray::new_timestamp( buffer![0i64, 86400000000, 172800000000].into_array(), TimeUnit::Microseconds, Some("UTC".into()), - )).unwrap()).unwrap())] - #[case::datetime_nanos(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + )))] + #[case::datetime_nanos(dtp_from_temporal(TemporalArray::new_timestamp( buffer![0i64, 86400000000000].into_array(), TimeUnit::Nanoseconds, Some("UTC".into()), - )).unwrap()).unwrap())] + )))] // Nullable arrays - #[case::datetime_nullable_seconds(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + #[case::datetime_nullable_seconds(dtp_from_temporal(TemporalArray::new_timestamp( PrimitiveArray::from_option_iter([Some(0i64), None, Some(86400), Some(172800), None]).into_array(), TimeUnit::Seconds, Some("UTC".into()), - )).unwrap()).unwrap())] + )))] // Edge cases - #[case::datetime_single(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + #[case::datetime_single(dtp_from_temporal(TemporalArray::new_timestamp( buffer![1234567890i64].into_array(), TimeUnit::Seconds, Some("UTC".into()), - )).unwrap()).unwrap())] + )))] // Large arrays (> 1024 elements) - #[case::datetime_large(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + #[case::datetime_large(dtp_from_temporal(TemporalArray::new_timestamp( PrimitiveArray::from_iter((0..1500).map(|i| i as i64 * 86400)).into_array(), TimeUnit::Seconds, Some("UTC".into()), - )).unwrap()).unwrap())] + )))] // Different time patterns - #[case::datetime_with_subseconds(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + #[case::datetime_with_subseconds(dtp_from_temporal(TemporalArray::new_timestamp( buffer![123456789i64, 234567890, 345678901, 456789012, 567890123].into_array(), TimeUnit::Milliseconds, Some("UTC".into()), - )).unwrap()).unwrap())] + )))] fn test_datetime_parts_consistency(#[case] array: DateTimePartsArray) { test_array_consistency(&array.into_array()); diff --git a/encodings/datetime-parts/src/compute/rules.rs b/encodings/datetime-parts/src/compute/rules.rs index a50eb98f95c..92c1f877c08 100644 --- a/encodings/datetime-parts/src/compute/rules.rs +++ b/encodings/datetime-parts/src/compute/rules.rs @@ -191,6 +191,7 @@ mod tests { use vortex_buffer::Buffer; use super::*; + use crate::DateTimeParts; use crate::DateTimePartsArray; const SECONDS_PER_DAY: i64 = 86400; @@ -214,8 +215,8 @@ mod tests { time_unit, None, ); - DateTimePartsArray::try_from_data(DateTimePartsData::try_from(temporal).unwrap()) - .vortex_expect("DateTimePartsData is always valid") + DateTimeParts::try_from_temporal(temporal) + .vortex_expect("TemporalArray must produce valid DateTimeParts") } /// Create a constant timestamp scalar at midnight for the given day. @@ -347,7 +348,7 @@ mod tests { TimeUnit::Seconds, None, ); - let dtp = DateTimePartsData::try_from(temporal).unwrap(); + let dtp = DateTimeParts::try_from_temporal(temporal).unwrap(); let len = dtp.len(); // Compare against midnight constant diff --git a/encodings/datetime-parts/src/compute/take.rs b/encodings/datetime-parts/src/compute/take.rs index 436e7b2d38f..4cd214663c1 100644 --- a/encodings/datetime-parts/src/compute/take.rs +++ b/encodings/datetime-parts/src/compute/take.rs @@ -37,8 +37,10 @@ fn take_datetime_parts( }; if !taken_seconds.dtype().is_nullable() && !taken_subseconds.dtype().is_nullable() { - return Ok(DateTimeParts::try_new(dtype, taken_days, taken_seconds, taken_subseconds)? - .into_array()); + return Ok( + DateTimeParts::try_new(dtype, taken_days, taken_seconds, taken_subseconds)? + .into_array(), + ); } // DateTimePartsArray requires seconds and subseconds to be non-nullable. @@ -98,11 +100,11 @@ mod tests { use vortex_array::extension::datetime::TimeUnit; use vortex_buffer::buffer; + use crate::DateTimeParts; use crate::DateTimePartsArray; - use crate::DateTimePartsData; #[rstest] - #[case(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( buffer![ 0i64, 86_400_000, // 1 day in ms @@ -112,8 +114,8 @@ mod tests { ].into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap()).unwrap())] - #[case(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + )).unwrap())] + #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( PrimitiveArray::from_option_iter([ Some(0i64), None, @@ -123,12 +125,12 @@ mod tests { ]).into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap()).unwrap())] - #[case(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + )).unwrap())] + #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( buffer![86_400_000i64].into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap()).unwrap())] + )).unwrap())] fn test_take_datetime_parts_conformance(#[case] array: DateTimePartsArray) { test_take_conformance(&array.into_array()); } diff --git a/encodings/decimal-byte-parts/public-api.lock b/encodings/decimal-byte-parts/public-api.lock index 1a93f2602f4..38ad5da9570 100644 --- a/encodings/decimal-byte-parts/public-api.lock +++ b/encodings/decimal-byte-parts/public-api.lock @@ -20,8 +20,6 @@ impl vortex_array::array::vtable::VTable for vortex_decimal_byte_parts::DecimalB pub type vortex_decimal_byte_parts::DecimalByteParts::ArrayData = vortex_decimal_byte_parts::DecimalBytePartsData -pub type vortex_decimal_byte_parts::DecimalByteParts::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_decimal_byte_parts::DecimalByteParts::OperationsVTable = vortex_decimal_byte_parts::DecimalByteParts pub type vortex_decimal_byte_parts::DecimalByteParts::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -34,11 +32,7 @@ pub fn vortex_decimal_byte_parts::DecimalByteParts::buffer(_array: vortex_array: pub fn vortex_decimal_byte_parts::DecimalByteParts::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_decimal_byte_parts::DecimalByteParts::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_decimal_byte_parts::DecimalByteParts::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_decimal_byte_parts::DecimalByteParts::dtype(array: &vortex_decimal_byte_parts::DecimalBytePartsData) -> &vortex_array::dtype::DType +pub fn vortex_decimal_byte_parts::DecimalByteParts::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_decimal_byte_parts::DecimalByteParts::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -46,23 +40,17 @@ pub fn vortex_decimal_byte_parts::DecimalByteParts::execute_parent(array: vortex pub fn vortex_decimal_byte_parts::DecimalByteParts::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_decimal_byte_parts::DecimalByteParts::len(array: &vortex_decimal_byte_parts::DecimalBytePartsData) -> usize - -pub fn vortex_decimal_byte_parts::DecimalByteParts::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_decimal_byte_parts::DecimalByteParts::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_decimal_byte_parts::DecimalByteParts::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_decimal_byte_parts::DecimalByteParts::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_decimal_byte_parts::DecimalByteParts::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_decimal_byte_parts::DecimalByteParts::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_decimal_byte_parts::DecimalByteParts::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_decimal_byte_parts::DecimalByteParts::stats(array: &vortex_decimal_byte_parts::DecimalBytePartsData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_decimal_byte_parts::DecimalByteParts::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_decimal_byte_parts::DecimalByteParts::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_decimal_byte_parts::DecimalByteParts::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -108,34 +96,22 @@ pub struct vortex_decimal_byte_parts::DecimalBytePartsData impl vortex_decimal_byte_parts::DecimalBytePartsData -pub fn vortex_decimal_byte_parts::DecimalBytePartsData::decimal_dtype(&self) -> &vortex_array::dtype::decimal::DecimalDType - -pub fn vortex_decimal_byte_parts::DecimalBytePartsData::dtype(&self) -> &vortex_array::dtype::DType - -pub fn vortex_decimal_byte_parts::DecimalBytePartsData::into_parts(self) -> vortex_decimal_byte_parts::DecimalBytePartsArrayParts - pub fn vortex_decimal_byte_parts::DecimalBytePartsData::is_empty(&self) -> bool pub fn vortex_decimal_byte_parts::DecimalBytePartsData::len(&self) -> usize pub fn vortex_decimal_byte_parts::DecimalBytePartsData::try_new(msp: vortex_array::array::erased::ArrayRef, decimal_dtype: vortex_array::dtype::decimal::DecimalDType) -> vortex_error::VortexResult +pub fn vortex_decimal_byte_parts::DecimalBytePartsData::validate(msp: &vortex_array::array::erased::ArrayRef, decimal_dtype: vortex_array::dtype::decimal::DecimalDType, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> + impl core::clone::Clone for vortex_decimal_byte_parts::DecimalBytePartsData pub fn vortex_decimal_byte_parts::DecimalBytePartsData::clone(&self) -> vortex_decimal_byte_parts::DecimalBytePartsData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_decimal_byte_parts::DecimalBytePartsData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_decimal_byte_parts::DecimalBytePartsData pub fn vortex_decimal_byte_parts::DecimalBytePartsData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_decimal_byte_parts::DecimalBytePartsData - -pub fn vortex_decimal_byte_parts::DecimalBytePartsData::into_array(self) -> vortex_array::array::erased::ArrayRef - pub struct vortex_decimal_byte_parts::DecimalBytesPartsMetadata impl vortex_decimal_byte_parts::DecimalBytesPartsMetadata diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index 606dae09f3e..fa62ac1ec02 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -40,11 +40,11 @@ pub fn vortex_fastlanes::bitpack_decompress::apply_patches_to_uninit_range_fn usize -pub fn vortex_fastlanes::bitpack_decompress::unpack_array(array: &vortex_fastlanes::BitPackedData, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fastlanes::bitpack_decompress::unpack_array(array: vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_fastlanes::bitpack_decompress::unpack_primitive_array(array: &vortex_fastlanes::BitPackedData, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fastlanes::bitpack_decompress::unpack_primitive_array(array: vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_fastlanes::bitpack_decompress::unpack_single(array: &vortex_fastlanes::BitPackedData, index: usize) -> vortex_array::scalar::Scalar +pub fn vortex_fastlanes::bitpack_decompress::unpack_single(array: vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>, index: usize) -> vortex_array::scalar::Scalar pub unsafe fn vortex_fastlanes::bitpack_decompress::unpack_single_primitive(packed: &[T], bit_width: usize, index_to_decode: usize) -> T @@ -86,7 +86,7 @@ impl vortex_fastlanes::unpack_iter: pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::full_chunks(&mut self) -> vortex_fastlanes::unpack_iter::BitUnpackIterator<'_, T> -pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::new(array: &vortex_fastlanes::BitPackedData) -> Self +pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::new(array: &vortex_fastlanes::BitPackedData, len: usize) -> Self pub trait vortex_fastlanes::unpack_iter::BitPacked: vortex_array::dtype::ptype::PhysicalPType @@ -124,6 +124,10 @@ pub const vortex_fastlanes::BitPacked::ID: vortex_array::array::ArrayId pub fn vortex_fastlanes::BitPacked::encode(array: &vortex_array::array::erased::ArrayRef, bit_width: u8) -> vortex_error::VortexResult +pub fn vortex_fastlanes::BitPacked::into_parts(array: vortex_fastlanes::BitPackedArray) -> vortex_fastlanes::BitPackedArrayParts + +pub fn vortex_fastlanes::BitPacked::try_new(packed: vortex_array::buffer::BufferHandle, ptype: vortex_array::dtype::ptype::PType, validity: vortex_array::validity::Validity, patches: core::option::Option, bit_width: u8, len: usize, offset: u16) -> vortex_error::VortexResult + impl core::clone::Clone for vortex_fastlanes::BitPacked pub fn vortex_fastlanes::BitPacked::clone(&self) -> vortex_fastlanes::BitPacked @@ -136,8 +140,6 @@ impl vortex_array::array::vtable::VTable for vortex_fastlanes::BitPacked pub type vortex_fastlanes::BitPacked::ArrayData = vortex_fastlanes::BitPackedData -pub type vortex_fastlanes::BitPacked::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_fastlanes::BitPacked::OperationsVTable = vortex_fastlanes::BitPacked pub type vortex_fastlanes::BitPacked::ValidityVTable = vortex_fastlanes::BitPacked @@ -152,11 +154,7 @@ pub fn vortex_fastlanes::BitPacked::buffer(array: vortex_array::array::view::Arr pub fn vortex_fastlanes::BitPacked::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_fastlanes::BitPacked::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_fastlanes::BitPacked::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_fastlanes::BitPacked::dtype(array: &vortex_fastlanes::BitPackedData) -> &vortex_array::dtype::DType +pub fn vortex_fastlanes::BitPacked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_fastlanes::BitPacked::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -164,23 +162,17 @@ pub fn vortex_fastlanes::BitPacked::execute_parent(array: vortex_array::array::v pub fn vortex_fastlanes::BitPacked::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_fastlanes::BitPacked::len(array: &vortex_fastlanes::BitPackedData) -> usize - -pub fn vortex_fastlanes::BitPacked::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_fastlanes::BitPacked::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_fastlanes::BitPacked::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_fastlanes::BitPacked::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_fastlanes::BitPacked::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_fastlanes::BitPacked::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_fastlanes::BitPacked::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_fastlanes::BitPacked::stats(array: &vortex_fastlanes::BitPackedData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_fastlanes::BitPacked::vtable(_array: &vortex_fastlanes::BitPackedData) -> &Self +pub fn vortex_fastlanes::BitPacked::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_fastlanes::BitPacked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -228,15 +220,9 @@ impl vortex_fastlanes::BitPackedData pub fn vortex_fastlanes::BitPackedData::bit_width(&self) -> u8 -pub fn vortex_fastlanes::BitPackedData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_fastlanes::BitPackedData::encode(array: &vortex_array::array::erased::ArrayRef, bit_width: u8) -> vortex_error::VortexResult -pub fn vortex_fastlanes::BitPackedData::into_parts(self) -> vortex_fastlanes::BitPackedArrayParts - -pub fn vortex_fastlanes::BitPackedData::is_empty(&self) -> bool - -pub fn vortex_fastlanes::BitPackedData::len(&self) -> usize +pub fn vortex_fastlanes::BitPackedData::into_parts(self, len: usize, nullability: vortex_array::dtype::nullability::Nullability) -> vortex_fastlanes::BitPackedArrayParts pub fn vortex_fastlanes::BitPackedData::max_packed_value(&self) -> usize @@ -246,36 +232,28 @@ pub fn vortex_fastlanes::BitPackedData::packed(&self) -> &vortex_array::buffer:: pub fn vortex_fastlanes::BitPackedData::packed_slice(&self) -> &[T] -pub fn vortex_fastlanes::BitPackedData::patches(&self) -> core::option::Option +pub fn vortex_fastlanes::BitPackedData::patches(&self, len: usize) -> core::option::Option -pub fn vortex_fastlanes::BitPackedData::ptype(&self) -> vortex_array::dtype::ptype::PType +pub fn vortex_fastlanes::BitPackedData::ptype(&self, dtype: &vortex_array::dtype::DType) -> vortex_array::dtype::ptype::PType pub fn vortex_fastlanes::BitPackedData::replace_patches(&mut self, patches: core::option::Option) pub fn vortex_fastlanes::BitPackedData::try_new(packed: vortex_array::buffer::BufferHandle, ptype: vortex_array::dtype::ptype::PType, validity: vortex_array::validity::Validity, patches: core::option::Option, bit_width: u8, length: usize, offset: u16) -> vortex_error::VortexResult -pub fn vortex_fastlanes::BitPackedData::unpacked_chunks(&self) -> vortex_fastlanes::unpack_iter::BitUnpackedChunks +pub fn vortex_fastlanes::BitPackedData::unpacked_chunks(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_fastlanes::unpack_iter::BitUnpackedChunks -pub fn vortex_fastlanes::BitPackedData::validity(&self) -> vortex_array::validity::Validity +pub fn vortex_fastlanes::BitPackedData::validity(&self, nullability: vortex_array::dtype::nullability::Nullability) -> vortex_array::validity::Validity -pub fn vortex_fastlanes::BitPackedData::validity_mask(&self) -> vortex_mask::Mask +pub fn vortex_fastlanes::BitPackedData::validity_mask(&self, len: usize, nullability: vortex_array::dtype::nullability::Nullability) -> vortex_mask::Mask impl core::clone::Clone for vortex_fastlanes::BitPackedData pub fn vortex_fastlanes::BitPackedData::clone(&self) -> vortex_fastlanes::BitPackedData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_fastlanes::BitPackedData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_fastlanes::BitPackedData pub fn vortex_fastlanes::BitPackedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_fastlanes::BitPackedData - -pub fn vortex_fastlanes::BitPackedData::into_array(self) -> vortex_array::array::erased::ArrayRef - pub struct vortex_fastlanes::Delta impl vortex_fastlanes::Delta @@ -284,6 +262,8 @@ pub const vortex_fastlanes::Delta::ID: vortex_array::array::ArrayId pub fn vortex_fastlanes::Delta::try_from_primitive_array(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fastlanes::Delta::try_new(bases: vortex_array::array::erased::ArrayRef, deltas: vortex_array::array::erased::ArrayRef, offset: usize, len: usize) -> vortex_error::VortexResult + impl core::clone::Clone for vortex_fastlanes::Delta pub fn vortex_fastlanes::Delta::clone(&self) -> vortex_fastlanes::Delta @@ -296,8 +276,6 @@ impl vortex_array::array::vtable::VTable for vortex_fastlanes::Delta pub type vortex_fastlanes::Delta::ArrayData = vortex_fastlanes::DeltaData -pub type vortex_fastlanes::Delta::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_fastlanes::Delta::OperationsVTable = vortex_fastlanes::Delta pub type vortex_fastlanes::Delta::ValidityVTable = vortex_fastlanes::Delta @@ -310,33 +288,23 @@ pub fn vortex_fastlanes::Delta::buffer(_array: vortex_array::array::view::ArrayV pub fn vortex_fastlanes::Delta::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_fastlanes::Delta::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_fastlanes::Delta::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_fastlanes::Delta::dtype(array: &vortex_fastlanes::DeltaData) -> &vortex_array::dtype::DType +pub fn vortex_fastlanes::Delta::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_fastlanes::Delta::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_fastlanes::Delta::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_fastlanes::Delta::len(array: &vortex_fastlanes::DeltaData) -> usize - -pub fn vortex_fastlanes::Delta::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_fastlanes::Delta::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_fastlanes::Delta::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_fastlanes::Delta::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_fastlanes::Delta::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_fastlanes::Delta::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_fastlanes::Delta::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_fastlanes::Delta::stats(array: &vortex_fastlanes::DeltaData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_fastlanes::Delta::vtable(_array: &vortex_fastlanes::DeltaData) -> &Self +pub fn vortex_fastlanes::Delta::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_fastlanes::Delta::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -364,34 +332,18 @@ pub fn vortex_fastlanes::DeltaData::bases(&self) -> &vortex_array::array::erased pub fn vortex_fastlanes::DeltaData::deltas(&self) -> &vortex_array::array::erased::ArrayRef -pub fn vortex_fastlanes::DeltaData::dtype(&self) -> &vortex_array::dtype::DType - -pub fn vortex_fastlanes::DeltaData::is_empty(&self) -> bool - -pub fn vortex_fastlanes::DeltaData::len(&self) -> usize - pub fn vortex_fastlanes::DeltaData::offset(&self) -> usize -pub fn vortex_fastlanes::DeltaData::try_from_primitive_array(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult - pub fn vortex_fastlanes::DeltaData::try_new(bases: vortex_array::array::erased::ArrayRef, deltas: vortex_array::array::erased::ArrayRef, offset: usize, len: usize) -> vortex_error::VortexResult impl core::clone::Clone for vortex_fastlanes::DeltaData pub fn vortex_fastlanes::DeltaData::clone(&self) -> vortex_fastlanes::DeltaData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_fastlanes::DeltaData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_fastlanes::DeltaData pub fn vortex_fastlanes::DeltaData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_fastlanes::DeltaData - -pub fn vortex_fastlanes::DeltaData::into_array(self) -> vortex_array::array::erased::ArrayRef - pub struct vortex_fastlanes::FoR impl vortex_fastlanes::FoR @@ -414,8 +366,6 @@ impl vortex_array::array::vtable::VTable for vortex_fastlanes::FoR pub type vortex_fastlanes::FoR::ArrayData = vortex_fastlanes::FoRData -pub type vortex_fastlanes::FoR::Metadata = vortex_array::scalar::Scalar - pub type vortex_fastlanes::FoR::OperationsVTable = vortex_fastlanes::FoR pub type vortex_fastlanes::FoR::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -428,11 +378,7 @@ pub fn vortex_fastlanes::FoR::buffer(_array: vortex_array::array::view::ArrayVie pub fn vortex_fastlanes::FoR::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_fastlanes::FoR::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_fastlanes::FoR::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_fastlanes::FoR::dtype(array: &vortex_fastlanes::FoRData) -> &vortex_array::dtype::DType +pub fn vortex_fastlanes::FoR::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_fastlanes::FoR::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -440,23 +386,17 @@ pub fn vortex_fastlanes::FoR::execute_parent(array: vortex_array::array::view::A pub fn vortex_fastlanes::FoR::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_fastlanes::FoR::len(array: &vortex_fastlanes::FoRData) -> usize - -pub fn vortex_fastlanes::FoR::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_fastlanes::FoR::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_fastlanes::FoR::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_fastlanes::FoR::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_fastlanes::FoR::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_fastlanes::FoR::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_fastlanes::FoR::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_fastlanes::FoR::stats(array: &vortex_fastlanes::FoRData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_fastlanes::FoR::vtable(_array: &vortex_fastlanes::FoRData) -> &Self +pub fn vortex_fastlanes::FoR::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_fastlanes::FoR::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -504,8 +444,6 @@ pub fn vortex_fastlanes::FoRData::ptype(&self) -> vortex_array::dtype::ptype::PT pub fn vortex_fastlanes::FoRData::reference_scalar(&self) -> &vortex_array::scalar::Scalar -pub fn vortex_fastlanes::FoRData::try_new(encoded: vortex_array::array::erased::ArrayRef, reference: vortex_array::scalar::Scalar) -> vortex_error::VortexResult - impl vortex_fastlanes::FoRData pub fn vortex_fastlanes::FoRData::encode(array: vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult @@ -514,18 +452,10 @@ impl core::clone::Clone for vortex_fastlanes::FoRData pub fn vortex_fastlanes::FoRData::clone(&self) -> vortex_fastlanes::FoRData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_fastlanes::FoRData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_fastlanes::FoRData pub fn vortex_fastlanes::FoRData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_fastlanes::FoRData - -pub fn vortex_fastlanes::FoRData::into_array(self) -> vortex_array::array::erased::ArrayRef - pub struct vortex_fastlanes::RLE impl vortex_fastlanes::RLE @@ -534,7 +464,9 @@ pub const vortex_fastlanes::RLE::ID: vortex_array::array::ArrayId pub fn vortex_fastlanes::RLE::encode(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult -pub unsafe fn vortex_fastlanes::RLE::new_unchecked(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, dtype: vortex_array::dtype::DType, offset: usize, length: usize) -> vortex_fastlanes::RLEArray +pub unsafe fn vortex_fastlanes::RLE::new_unchecked(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> vortex_fastlanes::RLEArray + +pub fn vortex_fastlanes::RLE::try_new(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> vortex_error::VortexResult impl core::clone::Clone for vortex_fastlanes::RLE @@ -548,11 +480,9 @@ impl vortex_array::array::vtable::VTable for vortex_fastlanes::RLE pub type vortex_fastlanes::RLE::ArrayData = vortex_fastlanes::RLEData -pub type vortex_fastlanes::RLE::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_fastlanes::RLE::OperationsVTable = vortex_fastlanes::RLE -pub type vortex_fastlanes::RLE::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChildSliceHelper +pub type vortex_fastlanes::RLE::ValidityVTable = vortex_fastlanes::RLE pub fn vortex_fastlanes::RLE::array_eq(array: &vortex_fastlanes::RLEData, other: &vortex_fastlanes::RLEData, precision: vortex_array::hash::Precision) -> bool @@ -562,11 +492,7 @@ pub fn vortex_fastlanes::RLE::buffer(_array: vortex_array::array::view::ArrayVie pub fn vortex_fastlanes::RLE::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_fastlanes::RLE::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_fastlanes::RLE::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_fastlanes::RLE::dtype(array: &vortex_fastlanes::RLEData) -> &vortex_array::dtype::DType +pub fn vortex_fastlanes::RLE::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_fastlanes::RLE::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -574,23 +500,17 @@ pub fn vortex_fastlanes::RLE::execute_parent(array: vortex_array::array::view::A pub fn vortex_fastlanes::RLE::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_fastlanes::RLE::len(array: &vortex_fastlanes::RLEData) -> usize - -pub fn vortex_fastlanes::RLE::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_fastlanes::RLE::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_fastlanes::RLE::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_fastlanes::RLE::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_fastlanes::RLE::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_fastlanes::RLE::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_fastlanes::RLE::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_fastlanes::RLE::stats(array: &vortex_fastlanes::RLEData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_fastlanes::RLE::vtable(_array: &vortex_fastlanes::RLEData) -> &Self +pub fn vortex_fastlanes::RLE::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_fastlanes::RLE::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -598,9 +518,9 @@ impl vortex_array::array::vtable::operations::OperationsVTable, index: usize, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -impl vortex_array::array::vtable::validity::ValidityChild for vortex_fastlanes::RLE +impl vortex_array::array::vtable::validity::ValidityVTable for vortex_fastlanes::RLE -pub fn vortex_fastlanes::RLE::validity_child(array: &vortex_fastlanes::RLEData) -> &vortex_array::array::erased::ArrayRef +pub fn vortex_fastlanes::RLE::validity(array: vortex_array::array::view::ArrayView<'_, vortex_fastlanes::RLE>) -> vortex_error::VortexResult impl vortex_array::arrays::slice::SliceKernel for vortex_fastlanes::RLE @@ -614,15 +534,13 @@ pub struct vortex_fastlanes::RLEData impl vortex_fastlanes::RLEData -pub fn vortex_fastlanes::RLEData::dtype(&self) -> &vortex_array::dtype::DType - -pub fn vortex_fastlanes::RLEData::indices(&self) -> &vortex_array::array::erased::ArrayRef +pub fn vortex_fastlanes::RLEData::encode(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult -pub fn vortex_fastlanes::RLEData::is_empty(&self) -> bool +impl vortex_fastlanes::RLEData -pub fn vortex_fastlanes::RLEData::len(&self) -> usize +pub fn vortex_fastlanes::RLEData::indices(&self) -> &vortex_array::array::erased::ArrayRef -pub unsafe fn vortex_fastlanes::RLEData::new_unchecked(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, dtype: vortex_array::dtype::DType, offset: usize, length: usize) -> Self +pub unsafe fn vortex_fastlanes::RLEData::new_unchecked(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, offset: usize) -> Self pub fn vortex_fastlanes::RLEData::offset(&self) -> usize @@ -632,30 +550,14 @@ pub fn vortex_fastlanes::RLEData::values(&self) -> &vortex_array::array::erased: pub fn vortex_fastlanes::RLEData::values_idx_offsets(&self) -> &vortex_array::array::erased::ArrayRef -impl vortex_fastlanes::RLEData - -pub fn vortex_fastlanes::RLEData::encode(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult - impl core::clone::Clone for vortex_fastlanes::RLEData pub fn vortex_fastlanes::RLEData::clone(&self) -> vortex_fastlanes::RLEData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_fastlanes::RLEData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_fastlanes::RLEData pub fn vortex_fastlanes::RLEData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_fastlanes::RLEData - -pub fn vortex_fastlanes::RLEData::into_array(self) -> vortex_array::array::erased::ArrayRef - -impl vortex_array::array::vtable::validity::ValidityChildSliceHelper for vortex_fastlanes::RLEData - -pub fn vortex_fastlanes::RLEData::unsliced_child_and_slice(&self) -> (&vortex_array::array::erased::ArrayRef, usize, usize) - pub fn vortex_fastlanes::delta_compress(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<(vortex_array::arrays::primitive::vtable::PrimitiveArray, vortex_array::arrays::primitive::vtable::PrimitiveArray)> pub fn vortex_fastlanes::initialize(session: &mut vortex_session::VortexSession) diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index 6c87ded63ac..4290e5cb44a 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -423,6 +423,7 @@ mod test { use vortex_session::VortexSession; use super::*; + use crate::BitPackedData; use crate::bitpack_compress::test_harness::make_array; static SESSION: LazyLock = @@ -448,7 +449,7 @@ mod test { ); assert!(values.ptype().is_unsigned_int()); let compressed = BitPackedData::encode(&values.into_array(), 4).unwrap(); - assert!(compressed.patches().is_none()); + assert!(compressed.patches(compressed.len()).is_none()); assert_eq!( (0..(1 << 4)).collect::>(), compressed @@ -510,7 +511,7 @@ mod test { let array = PrimitiveArray::from_iter(values); let bitpacked = bitpack_encode(&array, 4, None).unwrap(); - let patches = bitpacked.patches().unwrap(); + let patches = bitpacked.patches(bitpacked.len()).unwrap(); let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); // chunk 0 (0-1023): patches at 100, 200 -> starts at patch index 0 @@ -533,7 +534,7 @@ mod test { let array = PrimitiveArray::from_iter(values); let bitpacked = bitpack_encode(&array, 4, None).unwrap(); - let patches = bitpacked.patches().unwrap(); + let patches = bitpacked.patches(bitpacked.len()).unwrap(); let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); assert_arrays_eq!(chunk_offsets, PrimitiveArray::from_iter([0u64, 2, 2])); @@ -552,7 +553,7 @@ mod test { let array = PrimitiveArray::from_iter(values); let bitpacked = bitpack_encode(&array, 4, None).unwrap(); - let patches = bitpacked.patches().unwrap(); + let patches = bitpacked.patches(bitpacked.len()).unwrap(); let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); // chunk 0 (0-1023): patches at 100, 200 -> starts at patch index 0 @@ -576,7 +577,7 @@ mod test { let array = PrimitiveArray::from_iter(values); let bitpacked = bitpack_encode(&array, 4, None).unwrap(); - let patches = bitpacked.patches().unwrap(); + let patches = bitpacked.patches(bitpacked.len()).unwrap(); let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); // Single chunk starting at patch index 0. diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs index 989fe10b619..2f985b497a3 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs @@ -182,6 +182,10 @@ mod tests { static SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); + fn unpack(bitpacked: &BitPackedArray) -> VortexResult { + unpack_array(bitpacked.as_view(), &mut SESSION.create_execution_ctx()) + } + fn compression_roundtrip(n: usize) { let values = PrimitiveArray::from_iter((0..n).map(|i| (i % 2047) as u16)); let compressed = BitPackedData::encode(&values.clone().into_array(), 11).unwrap(); @@ -192,7 +196,9 @@ mod tests { .iter() .enumerate() .for_each(|(i, v)| { - let scalar: u16 = (&unpack_single(&compressed, i)).try_into().unwrap(); + let scalar: u16 = (&unpack_single(compressed.as_view(), i)) + .try_into() + .unwrap(); assert_eq!(scalar, *v); }); } @@ -214,7 +220,7 @@ mod tests { fn test_all_zeros() -> VortexResult<()> { let zeros = buffer![0u16, 0, 0, 0].into_array().to_primitive(); let bitpacked = encode(&zeros, 0); - let actual = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; + let actual = unpack(&bitpacked)?; assert_arrays_eq!(actual, PrimitiveArray::from_iter([0u16, 0, 0, 0])); Ok(()) } @@ -223,7 +229,7 @@ mod tests { fn test_simple_patches() -> VortexResult<()> { let zeros = buffer![0u16, 1, 0, 1].into_array().to_primitive(); let bitpacked = encode(&zeros, 0); - let actual = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; + let actual = unpack(&bitpacked)?; assert_arrays_eq!(actual, PrimitiveArray::from_iter([0u16, 1, 0, 1])); Ok(()) } @@ -232,7 +238,7 @@ mod tests { fn test_one_full_chunk() -> VortexResult<()> { let zeros = BufferMut::from_iter(0u16..1024).into_array().to_primitive(); let bitpacked = encode(&zeros, 10); - let actual = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; + let actual = unpack(&bitpacked)?; assert_arrays_eq!(actual, PrimitiveArray::from_iter(0u16..1024)); Ok(()) } @@ -243,8 +249,8 @@ mod tests { .into_array() .to_primitive(); let bitpacked = encode(&zeros, 10); - assert!(bitpacked.patches().is_some()); - let actual = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; + assert!(bitpacked.patches(bitpacked.len()).is_some()); + let actual = unpack(&bitpacked)?; assert_arrays_eq!( actual, PrimitiveArray::from_iter((5u16..1029).chain(5u16..1029).chain(5u16..1029)) @@ -256,8 +262,8 @@ mod tests { fn test_one_full_chunk_and_one_short_chunk_no_patch() -> VortexResult<()> { let zeros = BufferMut::from_iter(0u16..1025).into_array().to_primitive(); let bitpacked = encode(&zeros, 11); - assert!(bitpacked.patches().is_none()); - let actual = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; + assert!(bitpacked.patches(bitpacked.len()).is_none()); + let actual = unpack(&bitpacked)?; assert_arrays_eq!(actual, PrimitiveArray::from_iter(0u16..1025)); Ok(()) } @@ -269,8 +275,8 @@ mod tests { .to_primitive(); let bitpacked = encode(&zeros, 10); assert_eq!(bitpacked.len(), 1025); - assert!(bitpacked.patches().is_some()); - let actual = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; + assert!(bitpacked.patches(bitpacked.len()).is_some()); + let actual = unpack(&bitpacked)?; assert_arrays_eq!(actual, PrimitiveArray::from_iter(512u16..1537)); Ok(()) } @@ -282,7 +288,7 @@ mod tests { .to_primitive(); let bitpacked = encode(&zeros, 10); assert_eq!(bitpacked.len(), 1025); - assert!(bitpacked.patches().is_some()); + assert!(bitpacked.patches(bitpacked.len()).is_some()); let slice_ref = bitpacked.into_array().slice(1023..1025).unwrap(); let actual = { let mut ctx = SESSION.create_execution_ctx(); @@ -302,7 +308,7 @@ mod tests { .to_primitive(); let bitpacked = encode(&zeros, 10); assert_eq!(bitpacked.len(), 2229); - assert!(bitpacked.patches().is_some()); + assert!(bitpacked.patches(bitpacked.len()).is_some()); let slice_ref = bitpacked.into_array().slice(1023..2049).unwrap(); let actual = { let mut ctx = SESSION.create_execution_ctx(); @@ -325,7 +331,7 @@ mod tests { let mut builder = PrimitiveBuilder::::new(Nullability::NonNullable); unpack_into_primitive_builder( - &bitpacked, + &bitpacked.as_view(), &mut builder, &mut SESSION.create_execution_ctx(), )?; @@ -353,7 +359,7 @@ mod tests { // Unpack into a new builder. let mut builder = PrimitiveBuilder::::with_capacity(Nullability::Nullable, 5); unpack_into_primitive_builder( - &bitpacked, + &bitpacked.as_view(), &mut builder, &mut SESSION.create_execution_ctx(), )?; @@ -382,14 +388,14 @@ mod tests { // Bitpack with a bit width that will require patches. let bitpacked = encode(&array, 4); assert!( - bitpacked.patches().is_some(), + bitpacked.patches(bitpacked.len()).is_some(), "Should have patches for values > 15" ); // Unpack into a new builder. let mut builder = PrimitiveBuilder::::with_capacity(Nullability::NonNullable, 100); unpack_into_primitive_builder( - &bitpacked, + &bitpacked.as_view(), &mut builder, &mut SESSION.create_execution_ctx(), )?; @@ -414,7 +420,10 @@ mod tests { // Bitpack with a small bit width to force patches. let bitpacked = encode(&array, 6); - assert!(bitpacked.patches().is_some(), "Should have patches"); + assert!( + bitpacked.patches(bitpacked.len()).is_some(), + "Should have patches" + ); // Test with a larger array with multiple patches across chunks. let large_values: Vec = (0..3072) @@ -428,9 +437,9 @@ mod tests { .collect(); let large_array = PrimitiveArray::from_iter(large_values); let large_bitpacked = encode(&large_array, 8); - assert!(large_bitpacked.patches().is_some()); + assert!(large_bitpacked.patches(large_bitpacked.len()).is_some()); - let large_result = unpack_array(&large_bitpacked, &mut SESSION.create_execution_ctx())?; + let large_result = unpack(&large_bitpacked)?; assert_eq!(large_result.len(), 3072); Ok(()) } @@ -444,8 +453,7 @@ mod tests { let array = PrimitiveArray::new(values, validity); let bitpacked = encode(&array, 9); - let result = - unpack_array(&bitpacked, &mut SESSION.create_execution_ctx()).vortex_expect("unpack"); + let result = unpack(&bitpacked).vortex_expect("unpack"); // Verify length. assert_eq!(result.len(), 7); @@ -460,10 +468,9 @@ mod tests { let patch_array = PrimitiveArray::new(patch_values, patch_validity); let patch_bitpacked = encode(&patch_array, 5); - assert!(patch_bitpacked.patches().is_some()); + assert!(patch_bitpacked.patches(patch_bitpacked.len()).is_some()); - let patch_result = unpack_array(&patch_bitpacked, &mut SESSION.create_execution_ctx()) - .vortex_expect("unpack"); + let patch_result = unpack(&patch_bitpacked).vortex_expect("unpack"); assert_eq!(patch_result.len(), 7); // Test all nulls edge case. @@ -472,8 +479,7 @@ mod tests { Validity::from_iter([false, false, false, false]), ); let all_nulls_bp = encode(&all_nulls, 0); - let all_nulls_result = unpack_array(&all_nulls_bp, &mut SESSION.create_execution_ctx()) - .vortex_expect("unpack"); + let all_nulls_result = unpack(&all_nulls_bp).vortex_expect("unpack"); assert_eq!(all_nulls_result.len(), 4); } @@ -484,7 +490,7 @@ mod tests { let test_consistency = |array: &PrimitiveArray, bit_width: u8| -> VortexResult<()> { let bitpacked = encode(array, bit_width); - let unpacked_array = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; + let unpacked_array = unpack(&bitpacked)?; let executed = { let mut ctx = SESSION.create_execution_ctx(); @@ -588,23 +594,23 @@ mod tests { // Empty array. let empty: PrimitiveArray = PrimitiveArray::from_iter(Vec::::new()); let empty_bp = encode(&empty, 0); - let empty_result = unpack_array(&empty_bp, &mut SESSION.create_execution_ctx())?; + let empty_result = unpack(&empty_bp)?; assert_eq!(empty_result.len(), 0); // All zeros (bit_width = 0). let zeros = PrimitiveArray::from_iter([0u32; 100]); let zeros_bp = encode(&zeros, 0); - let zeros_result = unpack_array(&zeros_bp, &mut SESSION.create_execution_ctx())?; + let zeros_result = unpack(&zeros_bp)?; assert_eq!(zeros_result.len(), 100); // Verify consistency with unpack_array. - let zeros_array = unpack_array(&zeros_bp, &mut SESSION.create_execution_ctx())?; + let zeros_array = unpack(&zeros_bp)?; assert_eq!(zeros_result.len(), zeros_array.len()); assert_arrays_eq!(zeros_result, zeros_array); // Maximum bit width for u16 (15 bits, since bitpacking requires bit_width < type bit width). let max_values = PrimitiveArray::from_iter([32767u16; 50]); // 2^15 - 1 let max_bp = encode(&max_values, 15); - let max_result = unpack_array(&max_bp, &mut SESSION.create_execution_ctx())?; + let max_result = unpack(&max_bp)?; assert_eq!(max_result.len(), 50); // Exactly 3072 elements with patches across chunks. @@ -619,19 +625,19 @@ mod tests { .collect(); let boundary_array = PrimitiveArray::from_iter(boundary_values); let boundary_bp = encode(&boundary_array, 7); - assert!(boundary_bp.patches().is_some()); + assert!(boundary_bp.patches(boundary_bp.len()).is_some()); - let boundary_result = unpack_array(&boundary_bp, &mut SESSION.create_execution_ctx())?; + let boundary_result = unpack(&boundary_bp)?; assert_eq!(boundary_result.len(), 3072); // Verify consistency. - let boundary_unpacked = unpack_array(&boundary_bp, &mut SESSION.create_execution_ctx())?; + let boundary_unpacked = unpack(&boundary_bp)?; assert_eq!(boundary_result.len(), boundary_unpacked.len()); assert_arrays_eq!(boundary_result, boundary_unpacked); // Single element. let single = PrimitiveArray::from_iter([42u8]); let single_bp = encode(&single, 6); - let single_result = unpack_array(&single_bp, &mut SESSION.create_execution_ctx())?; + let single_result = unpack(&single_bp)?; assert_eq!(single_result.len(), 1); Ok(()) } diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index 00b488a96ca..26ec86da064 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -436,7 +436,11 @@ mod test { let parray = values.clone().into_array(); let packed_with_patches = BitPackedData::encode(&parray, 9).unwrap(); - assert!(packed_with_patches.patches().is_some()); + assert!( + packed_with_patches + .patches(packed_with_patches.len()) + .is_some() + ); assert_arrays_eq!( packed_with_patches.as_array().to_primitive(), PrimitiveArray::new(values, vortex_array::validity::Validity::NonNullable) diff --git a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs index e8d8ac059bd..8781a107e4d 100644 --- a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs +++ b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs @@ -58,7 +58,8 @@ impl> UnpackStrategy for BitPackingStr /// use vortex_fastlanes::unpack_iter::BitUnpackedChunks; /// /// let array = BitPackedData::encode(&buffer![2, 3, 4, 5].into_array(), 2).unwrap(); -/// let mut unpacked_chunks: BitUnpackedChunks = array.unpacked_chunks(); +/// let mut unpacked_chunks: BitUnpackedChunks = +/// array.unpacked_chunks(array.dtype(), array.len()); /// /// if let Some(header) = unpacked_chunks.initial() { /// // handle partial initial chunk diff --git a/encodings/fastlanes/src/bitpacking/compute/filter.rs b/encodings/fastlanes/src/bitpacking/compute/filter.rs index b8a7f398518..52987be657c 100644 --- a/encodings/fastlanes/src/bitpacking/compute/filter.rs +++ b/encodings/fastlanes/src/bitpacking/compute/filter.rs @@ -63,20 +63,21 @@ impl FilterKernel for BitPacked { } // Filter and patch using the correct unsigned type for FastLanes, then cast to signed if needed. - let primitive = match_each_unsigned_integer_ptype!(array.dtype().as_ptype().to_unsigned(), |U| { - let (buffer, validity) = filter_primitive_without_patches::(&array, values)?; - // reinterpret_cast for signed types. - let primitive = PrimitiveArray::new(buffer, validity); - if array.dtype().as_ptype().is_signed_int() { - PrimitiveArray::from_buffer_handle( - primitive.buffer_handle().clone(), - array.dtype().as_ptype(), - primitive.validity(), - ) - } else { - primitive - } - }); + let primitive = + match_each_unsigned_integer_ptype!(array.dtype().as_ptype().to_unsigned(), |U| { + let (buffer, validity) = filter_primitive_without_patches::(&array, values)?; + // reinterpret_cast for signed types. + let primitive = PrimitiveArray::new(buffer, validity); + if array.dtype().as_ptype().is_signed_int() { + PrimitiveArray::from_buffer_handle( + primitive.buffer_handle().clone(), + array.dtype().as_ptype(), + primitive.validity(), + ) + } else { + primitive + } + }); let patches = array .patches(array.len()) @@ -275,7 +276,7 @@ mod test { let unpacked = PrimitiveArray::from_iter(values.clone()); let bitpacked = BitPackedData::encode(&unpacked.into_array(), 7).unwrap(); assert!( - bitpacked.patches().is_some(), + bitpacked.patches(bitpacked.len()).is_some(), "Expected patches for values exceeding bit width" ); @@ -307,7 +308,7 @@ mod test { let unpacked = PrimitiveArray::from_iter(values.clone()); let bitpacked = BitPackedData::encode(&unpacked.into_array(), 7).unwrap(); assert!( - bitpacked.patches().is_some(), + bitpacked.patches(bitpacked.len()).is_some(), "Expected patches for values exceeding bit width" ); diff --git a/encodings/fastlanes/src/bitpacking/compute/slice.rs b/encodings/fastlanes/src/bitpacking/compute/slice.rs index e5977120a85..cc9394953f4 100644 --- a/encodings/fastlanes/src/bitpacking/compute/slice.rs +++ b/encodings/fastlanes/src/bitpacking/compute/slice.rs @@ -27,7 +27,9 @@ impl SliceReduce for BitPacked { BitPacked::try_new( array.packed().slice(encoded_start..encoded_stop), array.dtype().as_ptype(), - array.validity(array.dtype().nullability()).slice(range.clone())?, + array + .validity(array.dtype().nullability()) + .slice(range.clone())?, array .patches(array.len()) .map(|p| p.slice(range.clone())) diff --git a/encodings/fastlanes/src/bitpacking/compute/take.rs b/encodings/fastlanes/src/bitpacking/compute/take.rs index 807442b41df..a762d904323 100644 --- a/encodings/fastlanes/src/bitpacking/compute/take.rs +++ b/encodings/fastlanes/src/bitpacking/compute/take.rs @@ -57,7 +57,11 @@ impl TakeExecute for BitPacked { }) }); let taken = if ptype.is_signed_int() { - PrimitiveArray::from_buffer_handle(taken.buffer_handle().clone(), ptype, taken.validity()) + PrimitiveArray::from_buffer_handle( + taken.buffer_handle().clone(), + ptype, + taken.validity(), + ) } else { taken }; @@ -219,7 +223,7 @@ mod test { let values = (0..u16::MAX as u32 + num_patches as u32).collect::>(); let uncompressed = PrimitiveArray::new(values.clone(), Validity::NonNullable); let packed = BitPackedData::encode(&uncompressed.into_array(), 16).unwrap(); - assert!(packed.patches().is_some()); + assert!(packed.patches(packed.len()).is_some()); let rng = rng(); let range = Uniform::new(0, values.len()).unwrap(); @@ -251,7 +255,7 @@ mod test { BitPackedData::encode(&buffer![1i32, 2i32, 3i32, 4i32].into_array(), 1).unwrap(); let taken_primitive = take_primitive::( - &start, + &start.as_view(), &PrimitiveArray::from_iter([0u64, 1, 2, 3]), Validity::NonNullable, &mut LEGACY_SESSION.create_execution_ctx(), diff --git a/encodings/fastlanes/src/bitpacking/vtable/operations.rs b/encodings/fastlanes/src/bitpacking/vtable/operations.rs index 4c9850a91e3..49984c80ade 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/operations.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/operations.rs @@ -141,14 +141,14 @@ mod test { // We create an array that has 1 element that does not fit in the 6-bit range. let array = BitPackedData::encode(&buffer![0u32..=64].into_array(), 6).unwrap(); - assert!(array.patches().is_some()); + assert!(array.patches(array.len()).is_some()); - let patch_indices = array.patches().unwrap().indices().clone(); + let patch_indices = array.patches(array.len()).unwrap().indices().clone(); assert_eq!(patch_indices.len(), 1); // Slicing drops the empty patches array. let sliced_bp = slice_via_reduce(&array, 0..64); - assert!(sliced_bp.patches().is_none()); + assert!(sliced_bp.patches(sliced_bp.len()).is_none()); } #[test] @@ -177,30 +177,29 @@ mod test { #[test] fn scalar_at_invalid_patches() { - let packed_array = unsafe { - BitPackedData::new_unchecked( - BufferHandle::new_host(ByteBuffer::copy_from_aligned( - [0u8; 128], - Alignment::of::(), - )), - DType::Primitive(PType::U32, true.into()), - Validity::AllInvalid, - Some( - Patches::new( - 8, - 0, - buffer![1u32].into_array(), - PrimitiveArray::new(buffer![999u32], Validity::AllValid).into_array(), - None, - ) - .unwrap(), - ), - 1, - 8, - 0, - ) - .into_array() - }; + let packed_array = BitPacked::try_new( + BufferHandle::new_host(ByteBuffer::copy_from_aligned( + [0u8; 128], + Alignment::of::(), + )), + PType::U32, + Validity::AllInvalid, + Some( + Patches::new( + 8, + 0, + buffer![1u32].into_array(), + PrimitiveArray::new(buffer![999u32], Validity::AllValid).into_array(), + None, + ) + .unwrap(), + ), + 1, + 8, + 0, + ) + .unwrap() + .into_array(); assert_eq!( packed_array.scalar_at(1).unwrap(), Scalar::null(DType::Primitive(PType::U32, Nullability::Nullable)) @@ -212,9 +211,9 @@ mod test { let values = (0u32..257).collect::>(); let uncompressed = values.clone().into_array(); let packed = BitPackedData::encode(&uncompressed, 8).unwrap(); - assert!(packed.patches().is_some()); + assert!(packed.patches(packed.len()).is_some()); - let patches = packed.patches().unwrap().indices().clone(); + let patches = packed.patches(packed.len()).unwrap().indices().clone(); assert_eq!( usize::try_from(&patches.scalar_at(0).unwrap()).unwrap(), 256 diff --git a/encodings/fastlanes/src/delta/array/delta_compress.rs b/encodings/fastlanes/src/delta/array/delta_compress.rs index f12bf13b674..0b79489246d 100644 --- a/encodings/fastlanes/src/delta/array/delta_compress.rs +++ b/encodings/fastlanes/src/delta/array/delta_compress.rs @@ -105,7 +105,6 @@ mod tests { use vortex_session::VortexSession; use crate::Delta; - use crate::DeltaArray; use crate::bitpack_compress::bitpack_encode; use crate::delta::array::delta_decompress::delta_decompress; use crate::delta_compress; @@ -137,9 +136,13 @@ mod tests { ); let (bases, deltas) = delta_compress(&array, &mut SESSION.create_execution_ctx()).unwrap(); let bitpacked_deltas = bitpack_encode(&deltas, 1, None).unwrap(); - let packed_delta = - Delta::try_new(bases.into_array(), bitpacked_deltas.into_array(), 0, array.len()) - .vortex_expect("Delta array construction should succeed"); + let packed_delta = Delta::try_new( + bases.into_array(), + bitpacked_deltas.into_array(), + 0, + array.len(), + ) + .vortex_expect("Delta array construction should succeed"); assert_arrays_eq!(packed_delta.as_array().to_primitive(), array); } } diff --git a/encodings/fastlanes/src/delta/array/mod.rs b/encodings/fastlanes/src/delta/array/mod.rs index 8749f93d4c4..05d9dfbbadb 100644 --- a/encodings/fastlanes/src/delta/array/mod.rs +++ b/encodings/fastlanes/src/delta/array/mod.rs @@ -3,9 +3,6 @@ use fastlanes::FastLanes; use vortex_array::ArrayRef; -use vortex_array::ExecutionCtx; -use vortex_array::IntoArray; -use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_array::match_each_unsigned_integer_ptype; @@ -69,16 +66,6 @@ pub struct DeltaData { } impl DeltaData { - pub(crate) fn try_from_primitive_array( - array: &PrimitiveArray, - ctx: &mut ExecutionCtx, - ) -> VortexResult { - let logical_len = array.len(); - let (bases, deltas) = delta_compress::delta_compress(array, ctx)?; - - Self::try_new(bases.into_array(), deltas.into_array(), 0, logical_len) - } - /// Create a DeltaArray from the given `bases` and `deltas` arrays /// with given `offset` into first chunk and `logical_len` length. pub fn try_new( diff --git a/encodings/fastlanes/src/delta/compute/cast.rs b/encodings/fastlanes/src/delta/compute/cast.rs index bd1d4c29311..a9af6d09d60 100644 --- a/encodings/fastlanes/src/delta/compute/cast.rs +++ b/encodings/fastlanes/src/delta/compute/cast.rs @@ -37,8 +37,7 @@ impl CastReduce for Delta { // Create a new DeltaArray with the casted components, preserving offset and logical length Ok(Some( - Delta::try_new(casted_bases, casted_deltas, array.offset(), array.len())? - .into_array(), + Delta::try_new(casted_bases, casted_deltas, array.offset(), array.len())?.into_array(), )) } } @@ -68,8 +67,9 @@ mod tests { #[test] fn test_cast_delta_u8_to_u32() { let primitive = PrimitiveArray::from_iter([10u8, 20, 30, 40, 50]); - let array = Delta::try_from_primitive_array(&primitive, &mut SESSION.create_execution_ctx()) - .unwrap(); + let array = + Delta::try_from_primitive_array(&primitive, &mut SESSION.create_execution_ctx()) + .unwrap(); let casted = array .into_array() @@ -92,8 +92,8 @@ mod tests { buffer![100u16, 0, 200, 300, 0], vortex_array::validity::Validity::NonNullable, ); - let array = Delta::try_from_primitive_array(&values, &mut SESSION.create_execution_ctx()) - .unwrap(); + let array = + Delta::try_from_primitive_array(&values, &mut SESSION.create_execution_ctx()).unwrap(); let casted = array .into_array() diff --git a/encodings/fastlanes/src/delta/vtable/slice.rs b/encodings/fastlanes/src/delta/vtable/slice.rs index d479a5ab288..c836bed33fd 100644 --- a/encodings/fastlanes/src/delta/vtable/slice.rs +++ b/encodings/fastlanes/src/delta/vtable/slice.rs @@ -34,8 +34,7 @@ impl SliceReduce for Delta { )?; Ok(Some( - Delta::try_new(new_bases, new_deltas, physical_start % 1024, range.len())? - .into_array(), + Delta::try_new(new_bases, new_deltas, physical_start % 1024, range.len())?.into_array(), )) } } diff --git a/encodings/fastlanes/src/for/array/for_compress.rs b/encodings/fastlanes/src/for/array/for_compress.rs index 8bf8a8cd314..6bd4b36703a 100644 --- a/encodings/fastlanes/src/for/array/for_compress.rs +++ b/encodings/fastlanes/src/for/array/for_compress.rs @@ -71,7 +71,6 @@ mod test { use super::*; use crate::BitPackedData; - use crate::FoRArray; use crate::r#for::array::for_decompress::decompress; use crate::r#for::array::for_decompress::fused_decompress; @@ -174,7 +173,7 @@ mod test { .reinterpret_cast(PType::U8); let unsigned: Vec = (0..=u8::MAX).collect_vec(); let expected_unsigned = PrimitiveArray::from_iter(unsigned); - assert_arrays_eq!(encoded, expected_unsigned); + assert_eq!(encoded.as_slice::(), expected_unsigned.as_slice::()); let decompressed = decompress(&compressed, &mut SESSION.create_execution_ctx())?; array diff --git a/encodings/fastlanes/src/for/array/mod.rs b/encodings/fastlanes/src/for/array/mod.rs index 42b3b4a5230..f2cd3108250 100644 --- a/encodings/fastlanes/src/for/array/mod.rs +++ b/encodings/fastlanes/src/for/array/mod.rs @@ -37,13 +37,6 @@ impl FoRData { }) } - pub(crate) unsafe fn new_unchecked(encoded: ArrayRef, reference: Scalar) -> Self { - Self { - slots: vec![Some(encoded)], - reference, - } - } - pub(crate) fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { Self::validate_parts(self.encoded(), &self.reference, dtype, len) } @@ -54,10 +47,7 @@ impl FoRData { dtype: &DType, len: usize, ) -> VortexResult<()> { - vortex_ensure!( - !reference.is_null(), - "Reference value cannot be null" - ); + vortex_ensure!(!reference.is_null(), "Reference value cannot be null"); vortex_ensure!(dtype.is_int(), "FoR requires an integer dtype, got {dtype}"); vortex_ensure!( reference.dtype() == dtype, diff --git a/encodings/fastlanes/src/for/compute/cast.rs b/encodings/fastlanes/src/for/compute/cast.rs index e990a012d5d..e37b25b64a1 100644 --- a/encodings/fastlanes/src/for/compute/cast.rs +++ b/encodings/fastlanes/src/for/compute/cast.rs @@ -21,7 +21,9 @@ impl CastReduce for FoR { let casted_child = array.encoded().cast(dtype.clone())?; let casted_reference = array.reference_scalar().cast(dtype)?; - Ok(Some(FoR::try_new(casted_child, casted_reference)?.into_array())) + Ok(Some( + FoR::try_new(casted_child, casted_reference)?.into_array(), + )) } } diff --git a/encodings/fastlanes/src/for/compute/compare.rs b/encodings/fastlanes/src/for/compute/compare.rs index 93d25c7c846..853bf4c2955 100644 --- a/encodings/fastlanes/src/for/compute/compare.rs +++ b/encodings/fastlanes/src/for/compute/compare.rs @@ -80,7 +80,10 @@ where let rhs = Scalar::primitive(rhs, nullability); lhs.encoded() - .binary(ConstantArray::new(rhs, lhs.len()).into_array(), Operator::from(operator)) + .binary( + ConstantArray::new(rhs, lhs.len()).into_array(), + Operator::from(operator), + ) .map(Some) } @@ -117,8 +120,8 @@ mod tests { Nullability::NonNullable, CompareOperator::Eq, ) - .unwrap() - .unwrap(); + .unwrap() + .unwrap(); assert_arrays_eq!(result, BoolArray::from_iter([false, true, false].map(Some))); let result = compare_constant( @@ -161,9 +164,9 @@ mod tests { Nullability::Nullable, CompareOperator::Eq, ) - .unwrap() - .unwrap() - .dtype(), + .unwrap() + .unwrap() + .dtype(), &DType::Bool(Nullability::Nullable) ); assert_eq!( @@ -173,9 +176,9 @@ mod tests { Nullability::NonNullable, CompareOperator::Eq, ) - .unwrap() - .unwrap() - .dtype(), + .unwrap() + .unwrap() + .dtype(), &DType::Bool(Nullability::NonNullable) ); } @@ -195,8 +198,8 @@ mod tests { Nullability::NonNullable, CompareOperator::Eq, ) - .unwrap() - .unwrap(); + .unwrap() + .unwrap(); assert_arrays_eq!( result, BoolArray::from_iter([false, false, false].map(Some)) diff --git a/encodings/fastlanes/src/for/compute/mod.rs b/encodings/fastlanes/src/for/compute/mod.rs index 14f990e3ad6..8bf7e93d029 100644 --- a/encodings/fastlanes/src/for/compute/mod.rs +++ b/encodings/fastlanes/src/for/compute/mod.rs @@ -24,16 +24,22 @@ impl TakeExecute for FoR { _ctx: &mut ExecutionCtx, ) -> VortexResult> { Ok(Some( - FoR::try_new(array.encoded().take(indices.clone())?, array.reference_scalar().clone())? - .into_array(), + FoR::try_new( + array.encoded().take(indices.clone())?, + array.reference_scalar().clone(), + )? + .into_array(), )) } } impl FilterReduce for FoR { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { - FoR::try_new(array.encoded().filter(mask.clone())?, array.reference_scalar().clone()) - .map(|a| Some(a.into_array())) + FoR::try_new( + array.encoded().filter(mask.clone())?, + array.reference_scalar().clone(), + ) + .map(|a| Some(a.into_array())) } } @@ -48,8 +54,8 @@ mod test { use vortex_buffer::buffer; use vortex_error::VortexExpect; - use crate::FoRArray; use crate::FoR; + use crate::FoRArray; fn fa(encoded: ArrayRef, reference: Scalar) -> FoRArray { FoR::try_new(encoded, reference).vortex_expect("FoR array construction should succeed") @@ -102,8 +108,8 @@ mod tests { use vortex_buffer::buffer; use vortex_error::VortexExpect; - use crate::FoRArray; use crate::FoR; + use crate::FoRArray; fn fa(encoded: ArrayRef, reference: Scalar) -> FoRArray { FoR::try_new(encoded, reference).vortex_expect("FoR array construction should succeed") diff --git a/encodings/fastlanes/src/for/vtable/slice.rs b/encodings/fastlanes/src/for/vtable/slice.rs index 58914fad721..acd8a4d0254 100644 --- a/encodings/fastlanes/src/for/vtable/slice.rs +++ b/encodings/fastlanes/src/for/vtable/slice.rs @@ -14,8 +14,11 @@ use crate::FoR; impl SliceReduce for FoR { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { Ok(Some( - FoR::try_new(array.encoded().slice(range)?, array.reference_scalar().clone())? - .into_array(), + FoR::try_new( + array.encoded().slice(range)?, + array.reference_scalar().clone(), + )? + .into_array(), )) } } diff --git a/encodings/fastlanes/src/rle/array/mod.rs b/encodings/fastlanes/src/rle/array/mod.rs index 7d0459d85f8..2ee4a082dfa 100644 --- a/encodings/fastlanes/src/rle/array/mod.rs +++ b/encodings/fastlanes/src/rle/array/mod.rs @@ -247,7 +247,7 @@ mod tests { use vortex_session::registry::ReadContext; use crate::FL_CHUNK_SIZE; - use crate::RLEArray; + use crate::RLE; use crate::RLEData; use crate::test::SESSION; @@ -260,10 +260,8 @@ mod tests { PrimitiveArray::from_iter([0u16, 0, 1, 1, 2].iter().cycle().take(1024).copied()) .into_array(); let values_idx_offsets = PrimitiveArray::from_iter([0u64]).into_array(); - let rle_array = RLEArray::try_from_data( - RLEData::try_new(values, indices, values_idx_offsets, 0, 5).unwrap(), - ) - .vortex_expect("RLEData is always valid"); + let rle_array = RLE::try_new(values, indices, values_idx_offsets, 0, 5) + .vortex_expect("RLEData is always valid"); assert_eq!(rle_array.len(), 5); assert_eq!(rle_array.values().len(), 3); @@ -290,10 +288,8 @@ mod tests { ) .into_array(); - let rle_array = RLEArray::try_from_data( - RLEData::try_new(values, indices_with_validity, values_idx_offsets, 0, 3).unwrap(), - ) - .vortex_expect("RLEData is always valid"); + let rle_array = RLE::try_new(values, indices_with_validity, values_idx_offsets, 0, 3) + .vortex_expect("RLEData is always valid"); assert_eq!(rle_array.len(), 3); assert_eq!(rle_array.values().len(), 2); @@ -322,10 +318,8 @@ mod tests { ) .into_array(); - let rle_array = RLEArray::try_from_data( - RLEData::try_new(values, indices_with_validity, values_idx_offsets, 0, 5).unwrap(), - ) - .vortex_expect("RLEData is always valid"); + let rle_array = RLE::try_new(values, indices_with_validity, values_idx_offsets, 0, 5) + .vortex_expect("RLEData is always valid"); let valid_slice = rle_array.slice(0..3).unwrap().to_primitive(); // TODO(joe): replace with compute null count @@ -355,10 +349,8 @@ mod tests { ) .into_array(); - let rle_array = RLEArray::try_from_data( - RLEData::try_new(values, indices_with_validity, values_idx_offsets, 0, 5).unwrap(), - ) - .vortex_expect("RLEData is always valid"); + let rle_array = RLE::try_new(values, indices_with_validity, values_idx_offsets, 0, 5) + .vortex_expect("RLEData is always valid"); // TODO(joe): replace with compute null count let invalid_slice = rle_array @@ -393,10 +385,8 @@ mod tests { ) .into_array(); - let rle_array = RLEArray::try_from_data( - RLEData::try_new(values, indices_with_validity, values_idx_offsets, 0, 4).unwrap(), - ) - .vortex_expect("RLEData is always valid"); + let rle_array = RLE::try_new(values, indices_with_validity, values_idx_offsets, 0, 4) + .vortex_expect("RLEData is always valid"); let sliced_array = rle_array.slice(1..4).unwrap(); let validity_mask = sliced_array.validity_mask().unwrap(); @@ -416,15 +406,12 @@ mod tests { let values = PrimitiveArray::from_iter(Vec::::new()).into_array(); let indices = PrimitiveArray::from_iter(Vec::::new()).into_array(); let values_idx_offsets = PrimitiveArray::from_iter(Vec::::new()).into_array(); - let rle_array = RLEArray::try_from_data( - RLEData::try_new( - values, - indices.clone(), - values_idx_offsets, - 0, - indices.len(), - ) - .unwrap(), + let rle_array = RLE::try_new( + values, + indices.clone(), + values_idx_offsets, + 0, + indices.len(), ) .vortex_expect("RLEData is always valid"); @@ -437,10 +424,8 @@ mod tests { let values = PrimitiveArray::from_iter([10u32, 20, 30, 40]).into_array(); let indices = PrimitiveArray::from_iter([0u16, 1].repeat(1024)).into_array(); let values_idx_offsets = PrimitiveArray::from_iter([0u64, 2]).into_array(); - let rle_array = RLEArray::try_from_data( - RLEData::try_new(values, indices, values_idx_offsets, 0, 2048).unwrap(), - ) - .vortex_expect("RLEData is always valid"); + let rle_array = RLE::try_new(values, indices, values_idx_offsets, 0, 2048) + .vortex_expect("RLEData is always valid"); assert_eq!(rle_array.len(), 2048); assert_eq!(rle_array.values().len(), 4); @@ -489,15 +474,12 @@ mod tests { let primitive = PrimitiveArray::from_iter((0..2048).map(|i| (i / 100) as u32)); let rle_array = RLEData::encode(&primitive).unwrap(); - let sliced = RLEArray::try_from_data( - RLEData::try_new( - rle_array.values().clone(), - rle_array.indices().clone(), - rle_array.values_idx_offsets().clone(), - 100, - 100, - ) - .unwrap(), + let sliced = RLE::try_new( + rle_array.values().clone(), + rle_array.indices().clone(), + rle_array.values_idx_offsets().clone(), + 100, + 100, ) .vortex_expect("RLEData is always valid"); assert_eq!(sliced.len(), 100); @@ -558,16 +540,15 @@ mod tests { // Reconstruct the outer RLE with re-encoded indices. // SAFETY: we only replace the indices child; all other invariants hold. - let reconstructed = RLEArray::try_from_data(unsafe { - RLEData::new_unchecked( + let reconstructed = unsafe { + RLE::new_unchecked( rle.values().clone(), re_encoded.into_array(), rle.values_idx_offsets().clone(), - rle.dtype().clone(), rle.offset(), rle.len(), ) - })?; + }; // Decompress — panicked before the fill_forward_nulls chunk-boundary fix. let decoded = reconstructed.as_array().to_primitive(); diff --git a/encodings/fastlanes/src/rle/array/rle_compress.rs b/encodings/fastlanes/src/rle/array/rle_compress.rs index 74554d9deef..11270da5486 100644 --- a/encodings/fastlanes/src/rle/array/rle_compress.rs +++ b/encodings/fastlanes/src/rle/array/rle_compress.rs @@ -265,7 +265,7 @@ mod tests { #[case(vec![f16::ZERO, f16::NEG_ZERO])] #[case(vec![0f32, -0f32])] #[case(vec![0f64, -0f64])] - fn test_float_zeros(#[case] values: Vec) { + fn test_float_zeros(#[case] values: Vec) { let primitive = PrimitiveArray::from_iter(values); let rle = RLEData::encode(&primitive).unwrap(); let decoded = rle.as_array().to_primitive(); diff --git a/encodings/fastlanes/src/rle/compute/cast.rs b/encodings/fastlanes/src/rle/compute/cast.rs index 0d4777a8f9e..c3ae7d42e81 100644 --- a/encodings/fastlanes/src/rle/compute/cast.rs +++ b/encodings/fastlanes/src/rle/compute/cast.rs @@ -6,6 +6,7 @@ use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; @@ -13,7 +14,9 @@ use crate::rle::RLE; impl CastReduce for RLE { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // Cast RLE values. - let casted_values = array.values().cast(dtype.clone())?; + let casted_values = array + .values() + .cast(DType::Primitive(dtype.as_ptype(), Nullability::NonNullable))?; // Cast RLE indices such that validity matches the target dtype. let casted_indices = if array.indices().dtype().nullability() != dtype.nullability() { diff --git a/encodings/fastlanes/src/rle/vtable/operations.rs b/encodings/fastlanes/src/rle/vtable/operations.rs index 8804d30d178..e8dfc159f70 100644 --- a/encodings/fastlanes/src/rle/vtable/operations.rs +++ b/encodings/fastlanes/src/rle/vtable/operations.rs @@ -47,6 +47,7 @@ mod tests { use vortex_buffer::buffer; use super::*; + use crate::RLE; use crate::RLEArray; use crate::RLEData; @@ -65,15 +66,12 @@ mod tests { .into_array(); let values_idx_offsets = PrimitiveArray::from_iter([0u64]).into_array(); - RLEArray::try_from_data( - RLEData::try_new( - values, - indices.clone(), - values_idx_offsets, - 0, - indices.len(), - ) - .unwrap(), + RLE::try_new( + values, + indices.clone(), + values_idx_offsets, + 0, + indices.len(), ) .vortex_expect("RLEData is always valid") } @@ -103,15 +101,12 @@ mod tests { ) .into_array(); - RLEArray::try_from_data( - RLEData::try_new( - values, - indices.clone(), - values_idx_offsets, - 0, - indices.len(), - ) - .unwrap(), + RLE::try_new( + values, + indices.clone(), + values_idx_offsets, + 0, + indices.len(), ) .vortex_expect("RLEData is always valid") } diff --git a/encodings/fsst/public-api.lock b/encodings/fsst/public-api.lock index 32522dd3f04..4989f13dca7 100644 --- a/encodings/fsst/public-api.lock +++ b/encodings/fsst/public-api.lock @@ -20,8 +20,6 @@ impl vortex_array::array::vtable::VTable for vortex_fsst::FSST pub type vortex_fsst::FSST::ArrayData = vortex_fsst::FSSTData -pub type vortex_fsst::FSST::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_fsst::FSST::OperationsVTable = vortex_fsst::FSST pub type vortex_fsst::FSST::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -36,11 +34,7 @@ pub fn vortex_fsst::FSST::buffer(array: vortex_array::array::view::ArrayView<'_, pub fn vortex_fsst::FSST::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_fsst::FSST::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_fsst::FSST::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_fsst::FSST::dtype(array: &vortex_fsst::FSSTData) -> &vortex_array::dtype::DType +pub fn vortex_fsst::FSST::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_fsst::FSST::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -48,23 +42,17 @@ pub fn vortex_fsst::FSST::execute_parent(array: vortex_array::array::view::Array pub fn vortex_fsst::FSST::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_fsst::FSST::len(array: &vortex_fsst::FSSTData) -> usize - -pub fn vortex_fsst::FSST::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_fsst::FSST::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_fsst::FSST::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_fsst::FSST::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_fsst::FSST::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_fsst::FSST::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_fsst::FSST::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_fsst::FSST::stats(array: &vortex_fsst::FSSTData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_fsst::FSST::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_fsst::FSST::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_fsst::FSST::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -112,8 +100,6 @@ pub fn vortex_fsst::FSSTData::compressor(&self) -> &fsst::Compressor pub fn vortex_fsst::FSSTData::decompressor(&self) -> fsst::Decompressor<'_> -pub fn vortex_fsst::FSSTData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_fsst::FSSTData::is_empty(&self) -> bool pub fn vortex_fsst::FSSTData::len(&self) -> usize @@ -122,28 +108,22 @@ pub fn vortex_fsst::FSSTData::symbol_lengths(&self) -> &vortex_buffer::buffer::B pub fn vortex_fsst::FSSTData::symbols(&self) -> &vortex_buffer::buffer::Buffer -pub fn vortex_fsst::FSSTData::try_new(dtype: vortex_array::dtype::DType, symbols: vortex_buffer::buffer::Buffer, symbol_lengths: vortex_buffer::buffer::Buffer, codes: vortex_array::arrays::varbin::vtable::VarBinArray, uncompressed_lengths: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult +pub fn vortex_fsst::FSSTData::try_new(symbols: vortex_buffer::buffer::Buffer, symbol_lengths: vortex_buffer::buffer::Buffer, codes: vortex_array::arrays::varbin::vtable::VarBinArray, uncompressed_lengths: vortex_array::array::erased::ArrayRef, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult pub fn vortex_fsst::FSSTData::uncompressed_lengths(&self) -> &vortex_array::array::erased::ArrayRef pub fn vortex_fsst::FSSTData::uncompressed_lengths_dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_fsst::FSSTData::validate(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> + impl core::clone::Clone for vortex_fsst::FSSTData pub fn vortex_fsst::FSSTData::clone(&self) -> vortex_fsst::FSSTData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_fsst::FSSTData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_fsst::FSSTData pub fn vortex_fsst::FSSTData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_fsst::FSSTData - -pub fn vortex_fsst::FSSTData::into_array(self) -> vortex_array::array::erased::ArrayRef - pub struct vortex_fsst::FSSTMetadata impl vortex_fsst::FSSTMetadata diff --git a/encodings/pco/public-api.lock b/encodings/pco/public-api.lock index 21d9c41cec2..4513321e259 100644 --- a/encodings/pco/public-api.lock +++ b/encodings/pco/public-api.lock @@ -20,8 +20,6 @@ impl vortex_array::array::vtable::VTable for vortex_pco::Pco pub type vortex_pco::Pco::ArrayData = vortex_pco::PcoData -pub type vortex_pco::Pco::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_pco::Pco::OperationsVTable = vortex_pco::Pco pub type vortex_pco::Pco::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromValiditySliceHelper @@ -34,33 +32,23 @@ pub fn vortex_pco::Pco::buffer(array: vortex_array::array::view::ArrayView<'_, S pub fn vortex_pco::Pco::buffer_name(array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_pco::Pco::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_pco::Pco::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_pco::Pco::dtype(array: &vortex_pco::PcoData) -> &vortex_array::dtype::DType +pub fn vortex_pco::Pco::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_pco::Pco::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_pco::Pco::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_pco::Pco::len(array: &vortex_pco::PcoData) -> usize - -pub fn vortex_pco::Pco::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_pco::Pco::nbuffers(array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_pco::Pco::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_pco::Pco::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_pco::Pco::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_pco::Pco::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_pco::Pco::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_pco::Pco::stats(array: &vortex_pco::PcoData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_pco::Pco::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_pco::Pco::validate(&self, data: &vortex_pco::PcoData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_pco::Pco::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -104,8 +92,6 @@ impl vortex_pco::PcoData pub fn vortex_pco::PcoData::decompress(&self, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_pco::PcoData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_pco::PcoData::from_array(array: vortex_array::array::erased::ArrayRef, level: usize, nums_per_page: usize) -> vortex_error::VortexResult pub fn vortex_pco::PcoData::from_primitive(parray: &vortex_array::arrays::primitive::vtable::PrimitiveArray, level: usize, values_per_page: usize) -> vortex_error::VortexResult @@ -114,24 +100,18 @@ pub fn vortex_pco::PcoData::is_empty(&self) -> bool pub fn vortex_pco::PcoData::len(&self) -> usize -pub fn vortex_pco::PcoData::new(chunk_metas: alloc::vec::Vec, pages: alloc::vec::Vec, dtype: vortex_array::dtype::DType, metadata: vortex_pco::PcoMetadata, len: usize, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_pco::PcoData::new(chunk_metas: alloc::vec::Vec, pages: alloc::vec::Vec, ptype: vortex_array::dtype::ptype::PType, metadata: vortex_pco::PcoMetadata, len: usize, validity: vortex_array::validity::Validity) -> Self + +pub fn vortex_pco::PcoData::validate(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> impl core::clone::Clone for vortex_pco::PcoData pub fn vortex_pco::PcoData::clone(&self) -> vortex_pco::PcoData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_pco::PcoData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_pco::PcoData pub fn vortex_pco::PcoData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_pco::PcoData - -pub fn vortex_pco::PcoData::into_array(self) -> vortex_array::array::erased::ArrayRef - impl vortex_array::array::vtable::validity::ValiditySliceHelper for vortex_pco::PcoData pub fn vortex_pco::PcoData::unsliced_validity_and_slice(&self) -> (&vortex_array::validity::Validity, usize, usize) diff --git a/encodings/pco/src/compute/cast.rs b/encodings/pco/src/compute/cast.rs index 3b566eb390e..877535bbd97 100644 --- a/encodings/pco/src/compute/cast.rs +++ b/encodings/pco/src/compute/cast.rs @@ -39,10 +39,7 @@ impl CastReduce for Pco { ) ._slice(array.slice_start(), array.slice_stop()); - return Ok(Some( - Pco::try_new(dtype.clone(), data)? - .into_array(), - )); + return Ok(Some(Pco::try_new(dtype.clone(), data)?.into_array())); } // For other casts (e.g., numeric type changes), decode to canonical and let PrimitiveArray handle it diff --git a/encodings/pco/src/test.rs b/encodings/pco/src/test.rs index 647abaaa6aa..0bcbd8837ee 100644 --- a/encodings/pco/src/test.rs +++ b/encodings/pco/src/test.rs @@ -39,8 +39,6 @@ static SESSION: LazyLock = LazyLock::new(|| { }); use crate::Pco; -use crate::PcoArray; - #[test] fn test_compress_decompress() { let data: Vec = (0..200).collect(); @@ -101,7 +99,8 @@ fn test_validity_and_multiple_chunks_and_pages() { let compression_level = 3; let values_per_chunk = 33; let values_per_page = 10; - let compressed = PcoArray::try_from_data( + let compressed = Pco::try_new( + array.dtype().clone(), PcoData::from_primitive_with_values_per_chunk( &array, compression_level, diff --git a/encodings/runend/public-api.lock b/encodings/runend/public-api.lock index a3736617650..5822f213be4 100644 --- a/encodings/runend/public-api.lock +++ b/encodings/runend/public-api.lock @@ -44,8 +44,6 @@ impl vortex_array::array::vtable::VTable for vortex_runend::RunEnd pub type vortex_runend::RunEnd::ArrayData = vortex_runend::RunEndData -pub type vortex_runend::RunEnd::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_runend::RunEnd::OperationsVTable = vortex_runend::RunEnd pub type vortex_runend::RunEnd::ValidityVTable = vortex_runend::RunEnd @@ -58,11 +56,7 @@ pub fn vortex_runend::RunEnd::buffer(_array: vortex_array::array::view::ArrayVie pub fn vortex_runend::RunEnd::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_runend::RunEnd::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_runend::RunEnd::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_runend::RunEnd::dtype(array: &vortex_runend::RunEndData) -> &vortex_array::dtype::DType +pub fn vortex_runend::RunEnd::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_runend::RunEnd::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -70,23 +64,17 @@ pub fn vortex_runend::RunEnd::execute_parent(array: vortex_array::array::view::A pub fn vortex_runend::RunEnd::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_runend::RunEnd::len(array: &vortex_runend::RunEndData) -> usize - -pub fn vortex_runend::RunEnd::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_runend::RunEnd::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_runend::RunEnd::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_runend::RunEnd::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_runend::RunEnd::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_runend::RunEnd::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_runend::RunEnd::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_runend::RunEnd::stats(array: &vortex_runend::RunEndData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_runend::RunEnd::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_runend::RunEnd::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_runend::RunEnd::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -138,13 +126,9 @@ pub fn vortex_runend::RunEndData::find_physical_index(&self, index: usize) -> vo pub fn vortex_runend::RunEndData::into_parts(self) -> vortex_runend::RunEndArrayParts -pub fn vortex_runend::RunEndData::is_empty(&self) -> bool - -pub fn vortex_runend::RunEndData::len(&self) -> usize - pub fn vortex_runend::RunEndData::new(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef) -> Self -pub unsafe fn vortex_runend::RunEndData::new_unchecked(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> Self +pub unsafe fn vortex_runend::RunEndData::new_unchecked(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, _length: usize) -> Self pub fn vortex_runend::RunEndData::offset(&self) -> usize @@ -158,18 +142,10 @@ impl core::clone::Clone for vortex_runend::RunEndData pub fn vortex_runend::RunEndData::clone(&self) -> vortex_runend::RunEndData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_runend::RunEndData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_runend::RunEndData pub fn vortex_runend::RunEndData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_runend::RunEndData - -pub fn vortex_runend::RunEndData::into_array(self) -> vortex_array::array::erased::ArrayRef - impl vortex_array::arrow::FromArrowArray<&arrow_array::array::run_array::RunArray> for vortex_runend::RunEndData where ::Native: vortex_array::dtype::ptype::NativePType pub fn vortex_runend::RunEndData::from_arrow(array: &arrow_array::array::run_array::RunArray, nullable: bool) -> vortex_error::VortexResult diff --git a/encodings/runend/src/arrow.rs b/encodings/runend/src/arrow.rs index 674125f1f03..fd969c46c1c 100644 --- a/encodings/runend/src/arrow.rs +++ b/encodings/runend/src/arrow.rs @@ -30,9 +30,12 @@ where .reinterpret_cast(R::Native::PTYPE.to_unsigned()); let values = ArrayRef::from_arrow(array.values().as_ref(), nullable)?; - let ends_array = - PrimitiveArray::from_buffer_handle(ends.buffer_handle().clone(), ends.ptype(), ends.validity()) - .into_array(); + let ends_array = PrimitiveArray::from_buffer_handle( + ends.buffer_handle().clone(), + ends.ptype(), + ends.validity(), + ) + .into_array(); let (ends_slice, values_slice) = if offset == 0 && len == array.run_ends().max_value() { (ends_array, values) } else { diff --git a/encodings/sequence/public-api.lock b/encodings/sequence/public-api.lock index 95ccfef438c..c511a00a026 100644 --- a/encodings/sequence/public-api.lock +++ b/encodings/sequence/public-api.lock @@ -22,8 +22,6 @@ impl vortex_array::array::vtable::VTable for vortex_sequence::Sequence pub type vortex_sequence::Sequence::ArrayData = vortex_sequence::SequenceData -pub type vortex_sequence::Sequence::Metadata = vortex_sequence::array::SequenceMetadata - pub type vortex_sequence::Sequence::OperationsVTable = vortex_sequence::Sequence pub type vortex_sequence::Sequence::ValidityVTable = vortex_sequence::Sequence @@ -36,11 +34,7 @@ pub fn vortex_sequence::Sequence::buffer(_array: vortex_array::array::view::Arra pub fn vortex_sequence::Sequence::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_sequence::Sequence::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_sequence::Sequence::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_sequence::Sequence::dtype(array: &vortex_sequence::SequenceData) -> &vortex_array::dtype::DType +pub fn vortex_sequence::Sequence::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_sequence::Sequence::execute(array: vortex_array::array::typed::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -48,23 +42,17 @@ pub fn vortex_sequence::Sequence::execute_parent(array: vortex_array::array::vie pub fn vortex_sequence::Sequence::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_sequence::Sequence::len(array: &vortex_sequence::SequenceData) -> usize - -pub fn vortex_sequence::Sequence::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_sequence::Sequence::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_sequence::Sequence::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_sequence::Sequence::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_sequence::Sequence::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_sequence::Sequence::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_sequence::Sequence::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_sequence::Sequence::stats(array: &vortex_sequence::SequenceData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_sequence::Sequence::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_sequence::Sequence::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_sequence::Sequence::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -104,12 +92,8 @@ pub struct vortex_sequence::SequenceArrayParts pub vortex_sequence::SequenceArrayParts::base: vortex_array::scalar::typed_view::primitive::pvalue::PValue -pub vortex_sequence::SequenceArrayParts::len: usize - pub vortex_sequence::SequenceArrayParts::multiplier: vortex_array::scalar::typed_view::primitive::pvalue::PValue -pub vortex_sequence::SequenceArrayParts::nullability: vortex_array::dtype::nullability::Nullability - pub vortex_sequence::SequenceArrayParts::ptype: vortex_array::dtype::ptype::PType pub struct vortex_sequence::SequenceData @@ -118,16 +102,8 @@ impl vortex_sequence::SequenceData pub fn vortex_sequence::SequenceData::base(&self) -> vortex_array::scalar::typed_view::primitive::pvalue::PValue -pub fn vortex_sequence::SequenceData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_sequence::SequenceData::into_parts(self) -> vortex_sequence::SequenceArrayParts -pub fn vortex_sequence::SequenceData::is_empty(&self) -> bool - -pub fn vortex_sequence::SequenceData::last(&self) -> vortex_array::scalar::typed_view::primitive::pvalue::PValue - -pub fn vortex_sequence::SequenceData::len(&self) -> usize - pub fn vortex_sequence::SequenceData::multiplier(&self) -> vortex_array::scalar::typed_view::primitive::pvalue::PValue pub fn vortex_sequence::SequenceData::ptype(&self) -> vortex_array::dtype::ptype::PType @@ -136,22 +112,16 @@ pub fn vortex_sequence::SequenceData::try_new(base: vortex_array::scalar::typed_ pub fn vortex_sequence::SequenceData::try_new_typed>(base: T, multiplier: T, nullability: vortex_array::dtype::nullability::Nullability, length: usize) -> vortex_error::VortexResult +pub fn vortex_sequence::SequenceData::validate(base: vortex_array::scalar::typed_view::primitive::pvalue::PValue, multiplier: vortex_array::scalar::typed_view::primitive::pvalue::PValue, dtype: &vortex_array::dtype::DType, length: usize) -> vortex_error::VortexResult<()> + impl core::clone::Clone for vortex_sequence::SequenceData pub fn vortex_sequence::SequenceData::clone(&self) -> vortex_sequence::SequenceData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_sequence::SequenceData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_sequence::SequenceData pub fn vortex_sequence::SequenceData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_sequence::SequenceData - -pub fn vortex_sequence::SequenceData::into_array(self) -> vortex_array::array::erased::ArrayRef - pub fn vortex_sequence::initialize(session: &mut vortex_session::VortexSession) pub fn vortex_sequence::sequence_encode(primitive_array: &vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult> diff --git a/encodings/sparse/public-api.lock b/encodings/sparse/public-api.lock index 71b42ab8dbd..74d820647b4 100644 --- a/encodings/sparse/public-api.lock +++ b/encodings/sparse/public-api.lock @@ -30,6 +30,8 @@ pub fn vortex_sparse::Sparse::encode(array: &vortex_array::array::erased::ArrayR pub fn vortex_sparse::Sparse::try_new(indices: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, len: usize, fill_value: vortex_array::scalar::Scalar) -> vortex_error::VortexResult +pub fn vortex_sparse::Sparse::try_new_from_patches(patches: vortex_array::patches::Patches, fill_value: vortex_array::scalar::Scalar) -> vortex_error::VortexResult + impl core::clone::Clone for vortex_sparse::Sparse pub fn vortex_sparse::Sparse::clone(&self) -> vortex_sparse::Sparse @@ -42,8 +44,6 @@ impl vortex_array::array::vtable::VTable for vortex_sparse::Sparse pub type vortex_sparse::Sparse::ArrayData = vortex_sparse::SparseData -pub type vortex_sparse::Sparse::Metadata = vortex_sparse::SparseMetadata - pub type vortex_sparse::Sparse::OperationsVTable = vortex_sparse::Sparse pub type vortex_sparse::Sparse::ValidityVTable = vortex_sparse::Sparse @@ -56,11 +56,7 @@ pub fn vortex_sparse::Sparse::buffer(array: vortex_array::array::view::ArrayView pub fn vortex_sparse::Sparse::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_sparse::Sparse::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_sparse::Sparse::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_sparse::Sparse::dtype(array: &vortex_sparse::SparseData) -> &vortex_array::dtype::DType +pub fn vortex_sparse::Sparse::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_sparse::Sparse::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -68,23 +64,17 @@ pub fn vortex_sparse::Sparse::execute_parent(array: vortex_array::array::view::A pub fn vortex_sparse::Sparse::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_sparse::Sparse::len(array: &vortex_sparse::SparseData) -> usize - -pub fn vortex_sparse::Sparse::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_sparse::Sparse::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_sparse::Sparse::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_sparse::Sparse::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_sparse::Sparse::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_sparse::Sparse::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_sparse::Sparse::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_sparse::Sparse::stats(array: &vortex_sparse::SparseData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_sparse::Sparse::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_sparse::Sparse::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_sparse::Sparse::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -138,26 +128,14 @@ pub fn vortex_sparse::SparseData::try_new(indices: vortex_array::array::erased:: pub fn vortex_sparse::SparseData::try_new_from_patches(patches: vortex_array::patches::Patches, fill_value: vortex_array::scalar::Scalar) -> vortex_error::VortexResult +pub fn vortex_sparse::SparseData::validate(patches: &vortex_array::patches::Patches, fill_value: &vortex_array::scalar::Scalar, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> + impl core::clone::Clone for vortex_sparse::SparseData pub fn vortex_sparse::SparseData::clone(&self) -> vortex_sparse::SparseData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_sparse::SparseData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_sparse::SparseData pub fn vortex_sparse::SparseData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_sparse::SparseData - -pub fn vortex_sparse::SparseData::into_array(self) -> vortex_array::array::erased::ArrayRef - -pub struct vortex_sparse::SparseMetadata - -impl core::fmt::Debug for vortex_sparse::SparseMetadata - -pub fn vortex_sparse::SparseMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub type vortex_sparse::SparseArray = vortex_array::array::typed::Array diff --git a/encodings/sparse/src/compute/filter.rs b/encodings/sparse/src/compute/filter.rs index f804119dbf3..2cc8064a6cd 100644 --- a/encodings/sparse/src/compute/filter.rs +++ b/encodings/sparse/src/compute/filter.rs @@ -26,8 +26,7 @@ impl FilterKernel for Sparse { }; Ok(Some( - Sparse::try_new_from_patches(new_patches, array.fill_scalar().clone())? - .into_array(), + Sparse::try_new_from_patches(new_patches, array.fill_scalar().clone())?.into_array(), )) } } diff --git a/encodings/sparse/src/rules.rs b/encodings/sparse/src/rules.rs index f9c2832cb63..c53e7e4c6a5 100644 --- a/encodings/sparse/src/rules.rs +++ b/encodings/sparse/src/rules.rs @@ -22,6 +22,8 @@ impl NotReduce for Sparse { fn invert(array: ArrayView<'_, Self>) -> VortexResult> { let inverted_fill = array.fill_scalar().as_bool().invert().into_scalar(); let inverted_patches = array.patches().clone().map_values(|values| values.not())?; - Ok(Some(Sparse::try_new_from_patches(inverted_patches, inverted_fill)?.into_array())) + Ok(Some( + Sparse::try_new_from_patches(inverted_patches, inverted_fill)?.into_array(), + )) } } diff --git a/encodings/sparse/src/slice.rs b/encodings/sparse/src/slice.rs index 9952954a229..7a36d58e209 100644 --- a/encodings/sparse/src/slice.rs +++ b/encodings/sparse/src/slice.rs @@ -35,8 +35,7 @@ impl SliceKernel for Sparse { // patches slice will ensure that dtype of patches is unchanged and the indices and // values match Ok(Some( - unsafe { Sparse::new_unchecked(new_patches, array.fill_scalar().clone()) } - .into_array(), + unsafe { Sparse::new_unchecked(new_patches, array.fill_scalar().clone()) }.into_array(), )) } } diff --git a/encodings/zigzag/public-api.lock b/encodings/zigzag/public-api.lock index ab49a0acca7..ec8ed62f664 100644 --- a/encodings/zigzag/public-api.lock +++ b/encodings/zigzag/public-api.lock @@ -20,8 +20,6 @@ impl vortex_array::array::vtable::VTable for vortex_zigzag::ZigZag pub type vortex_zigzag::ZigZag::ArrayData = vortex_zigzag::ZigZagData -pub type vortex_zigzag::ZigZag::Metadata = vortex_array::metadata::EmptyMetadata - pub type vortex_zigzag::ZigZag::OperationsVTable = vortex_zigzag::ZigZag pub type vortex_zigzag::ZigZag::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -34,11 +32,7 @@ pub fn vortex_zigzag::ZigZag::buffer(_array: vortex_array::array::view::ArrayVie pub fn vortex_zigzag::ZigZag::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_zigzag::ZigZag::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_zigzag::ZigZag::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_zigzag::ZigZag::dtype(array: &vortex_zigzag::ZigZagData) -> &vortex_array::dtype::DType +pub fn vortex_zigzag::ZigZag::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_zigzag::ZigZag::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -46,23 +40,17 @@ pub fn vortex_zigzag::ZigZag::execute_parent(array: vortex_array::array::view::A pub fn vortex_zigzag::ZigZag::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_zigzag::ZigZag::len(array: &vortex_zigzag::ZigZagData) -> usize - -pub fn vortex_zigzag::ZigZag::metadata(_array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_zigzag::ZigZag::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_zigzag::ZigZag::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_zigzag::ZigZag::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_zigzag::ZigZag::serialize(_array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_zigzag::ZigZag::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_zigzag::ZigZag::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_zigzag::ZigZag::stats(array: &vortex_zigzag::ZigZagData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_zigzag::ZigZag::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_zigzag::ZigZag::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_zigzag::ZigZag::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -98,8 +86,6 @@ pub struct vortex_zigzag::ZigZagData impl vortex_zigzag::ZigZagData -pub fn vortex_zigzag::ZigZagData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_zigzag::ZigZagData::encoded(&self) -> &vortex_array::array::erased::ArrayRef pub fn vortex_zigzag::ZigZagData::is_empty(&self) -> bool @@ -116,18 +102,10 @@ impl core::clone::Clone for vortex_zigzag::ZigZagData pub fn vortex_zigzag::ZigZagData::clone(&self) -> vortex_zigzag::ZigZagData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_zigzag::ZigZagData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_zigzag::ZigZagData pub fn vortex_zigzag::ZigZagData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_zigzag::ZigZagData - -pub fn vortex_zigzag::ZigZagData::into_array(self) -> vortex_array::array::erased::ArrayRef - pub fn vortex_zigzag::zigzag_decode(parray: vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_array::arrays::primitive::vtable::PrimitiveArray pub fn vortex_zigzag::zigzag_encode(parray: vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult diff --git a/encodings/zigzag/src/slice.rs b/encodings/zigzag/src/slice.rs index 26bc92b820b..e027cda04cb 100644 --- a/encodings/zigzag/src/slice.rs +++ b/encodings/zigzag/src/slice.rs @@ -13,6 +13,8 @@ use crate::ZigZag; impl SliceReduce for ZigZag { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - Ok(Some(ZigZag::try_new(array.encoded().slice(range)?)?.into_array())) + Ok(Some( + ZigZag::try_new(array.encoded().slice(range)?)?.into_array(), + )) } } diff --git a/encodings/zstd/public-api.lock b/encodings/zstd/public-api.lock index 29554686839..85f0e4ebb05 100644 --- a/encodings/zstd/public-api.lock +++ b/encodings/zstd/public-api.lock @@ -6,12 +6,18 @@ impl vortex_zstd::Zstd pub const vortex_zstd::Zstd::ID: vortex_array::array::ArrayId +pub fn vortex_zstd::Zstd::decompress(array: &vortex_zstd::ZstdArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult + pub fn vortex_zstd::Zstd::from_primitive(parray: &vortex_array::arrays::primitive::vtable::PrimitiveArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult pub fn vortex_zstd::Zstd::from_var_bin_view(vbv: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult pub fn vortex_zstd::Zstd::from_var_bin_view_without_dict(vbv: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult +pub fn vortex_zstd::Zstd::into_parts(array: vortex_zstd::ZstdArray) -> vortex_zstd::ZstdArrayParts + +pub fn vortex_zstd::Zstd::try_new(dtype: vortex_array::dtype::DType, data: vortex_zstd::ZstdData) -> vortex_error::VortexResult + impl core::clone::Clone for vortex_zstd::Zstd pub fn vortex_zstd::Zstd::clone(&self) -> vortex_zstd::Zstd @@ -24,8 +30,6 @@ impl vortex_array::array::vtable::VTable for vortex_zstd::Zstd pub type vortex_zstd::Zstd::ArrayData = vortex_zstd::ZstdData -pub type vortex_zstd::Zstd::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_zstd::Zstd::OperationsVTable = vortex_zstd::Zstd pub type vortex_zstd::Zstd::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromValiditySliceHelper @@ -38,33 +42,23 @@ pub fn vortex_zstd::Zstd::buffer(array: vortex_array::array::view::ArrayView<'_, pub fn vortex_zstd::Zstd::buffer_name(array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_zstd::Zstd::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_zstd::Zstd::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_zstd::Zstd::dtype(array: &vortex_zstd::ZstdData) -> &vortex_array::dtype::DType +pub fn vortex_zstd::Zstd::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_zstd::Zstd::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_zstd::Zstd::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_zstd::Zstd::len(array: &vortex_zstd::ZstdData) -> usize - -pub fn vortex_zstd::Zstd::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_zstd::Zstd::nbuffers(array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_zstd::Zstd::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_zstd::Zstd::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_zstd::Zstd::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_zstd::Zstd::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_zstd::Zstd::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_zstd::Zstd::stats(array: &vortex_zstd::ZstdData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_zstd::Zstd::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_zstd::Zstd::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_zstd::Zstd::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -134,10 +128,6 @@ pub struct vortex_zstd::ZstdData impl vortex_zstd::ZstdData -pub fn vortex_zstd::ZstdData::decompress(&self, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult - -pub fn vortex_zstd::ZstdData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_zstd::ZstdData::from_array(array: vortex_array::array::erased::ArrayRef, level: i32, values_per_frame: usize) -> vortex_error::VortexResult pub fn vortex_zstd::ZstdData::from_canonical(canonical: &vortex_array::canonical::Canonical, level: i32, values_per_frame: usize) -> vortex_error::VortexResult> @@ -150,30 +140,22 @@ pub fn vortex_zstd::ZstdData::from_var_bin_view(vbv: &vortex_array::arrays::varb pub fn vortex_zstd::ZstdData::from_var_bin_view_without_dict(vbv: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult -pub fn vortex_zstd::ZstdData::into_parts(self) -> vortex_zstd::ZstdArrayParts - pub fn vortex_zstd::ZstdData::is_empty(&self) -> bool pub fn vortex_zstd::ZstdData::len(&self) -> usize -pub fn vortex_zstd::ZstdData::new(dictionary: core::option::Option, frames: alloc::vec::Vec, dtype: vortex_array::dtype::DType, metadata: vortex_zstd::ZstdMetadata, n_rows: usize, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_zstd::ZstdData::new(dictionary: core::option::Option, frames: alloc::vec::Vec, metadata: vortex_zstd::ZstdMetadata, n_rows: usize, validity: vortex_array::validity::Validity) -> Self + +pub fn vortex_zstd::ZstdData::validate(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> impl core::clone::Clone for vortex_zstd::ZstdData pub fn vortex_zstd::ZstdData::clone(&self) -> vortex_zstd::ZstdData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_zstd::ZstdData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_zstd::ZstdData pub fn vortex_zstd::ZstdData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_zstd::ZstdData - -pub fn vortex_zstd::ZstdData::into_array(self) -> vortex_array::array::erased::ArrayRef - impl vortex_array::array::vtable::validity::ValiditySliceHelper for vortex_zstd::ZstdData pub fn vortex_zstd::ZstdData::unsliced_validity_and_slice(&self) -> (&vortex_array::validity::Validity, usize, usize) diff --git a/encodings/zstd/src/slice.rs b/encodings/zstd/src/slice.rs index c5fd5d528ee..2f74d40072c 100644 --- a/encodings/zstd/src/slice.rs +++ b/encodings/zstd/src/slice.rs @@ -14,8 +14,11 @@ use crate::Zstd; impl SliceReduce for Zstd { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { Ok(Some( - Zstd::try_new(array.dtype().clone(), array.data().with_slice(range.start, range.end))? - .into_array(), + Zstd::try_new( + array.dtype().clone(), + array.data().with_slice(range.start, range.end), + )? + .into_array(), )) } } diff --git a/encodings/zstd/src/test.rs b/encodings/zstd/src/test.rs index f4f78b72ff7..7548f32cb80 100644 --- a/encodings/zstd/src/test.rs +++ b/encodings/zstd/src/test.rs @@ -78,7 +78,9 @@ fn test_zstd_with_validity_and_multi_frame() { assert_nth_scalar!(compressed, 177, 177); let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let decompressed = Zstd::decompress(&compressed, &mut ctx).unwrap().to_primitive(); + let decompressed = Zstd::decompress(&compressed, &mut ctx) + .unwrap() + .to_primitive(); let decompressed_values = decompressed.as_slice::(); assert_eq!(decompressed_values[3], 3); assert_eq!(decompressed_values[177], 177); @@ -121,7 +123,9 @@ fn test_zstd_with_dict() { assert_nth_scalar!(compressed, 199, 199); let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let decompressed = Zstd::decompress(&compressed, &mut ctx).unwrap().to_primitive(); + let decompressed = Zstd::decompress(&compressed, &mut ctx) + .unwrap() + .to_primitive(); assert_arrays_eq!(decompressed, PrimitiveArray::from_iter(data)); // check slicing works @@ -193,7 +197,9 @@ fn test_zstd_decompress_var_bin_view() { assert_nth_scalar!(compressed, 4, "baz"); let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let decompressed = Zstd::decompress(&compressed, &mut ctx).unwrap().to_varbinview(); + let decompressed = Zstd::decompress(&compressed, &mut ctx) + .unwrap() + .to_varbinview(); assert_nth_scalar!(decompressed, 0, "foo"); assert_nth_scalar!(decompressed, 1, "bar"); assert_nth_scalar!(decompressed, 2, None::); diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 5189514d877..68d709c8e51 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -864,8 +864,6 @@ impl vortex_array::VTable for vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ArrayData = vortex_array::arrays::bool::BoolData -pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Bool::OperationsVTable = vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool @@ -880,15 +878,11 @@ pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Bool::dtype(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -896,10 +890,6 @@ pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Bool::len(array: &vortex_array::arrays::bool::BoolData) -> usize - -pub fn vortex_array::arrays::Bool::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Bool::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -908,15 +898,13 @@ pub fn vortex_array::arrays::Bool::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Bool::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -968,7 +956,7 @@ pub struct vortex_array::arrays::bool::BoolData impl vortex_array::arrays::bool::BoolData -pub fn vortex_array::arrays::bool::BoolData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::bool::BoolData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::bool::BoolData::into_bit_buffer(self) -> vortex_buffer::bit::buf::BitBuffer @@ -994,18 +982,10 @@ impl core::clone::Clone for vortex_array::arrays::bool::BoolData pub fn vortex_array::arrays::bool::BoolData::clone(&self) -> vortex_array::arrays::bool::BoolData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::bool::BoolData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::bool::BoolData pub fn vortex_array::arrays::bool::BoolData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::bool::BoolData - -pub fn vortex_array::arrays::bool::BoolData::into_array(self) -> vortex_array::ArrayRef - pub struct vortex_array::arrays::bool::BoolMaskedValidityRule impl core::default::Default for vortex_array::arrays::bool::BoolMaskedValidityRule @@ -1048,8 +1028,6 @@ impl vortex_array::VTable for vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ArrayData = vortex_array::arrays::chunked::ChunkedData -pub type vortex_array::arrays::Chunked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Chunked::OperationsVTable = vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::Chunked @@ -1064,15 +1042,11 @@ pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Chunked::dtype(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -1080,10 +1054,6 @@ pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayV pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Chunked::len(array: &vortex_array::arrays::chunked::ChunkedData) -> usize - -pub fn vortex_array::arrays::Chunked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Chunked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1092,15 +1062,13 @@ pub fn vortex_array::arrays::Chunked::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Chunked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -1174,18 +1142,10 @@ impl core::clone::Clone for vortex_array::arrays::chunked::ChunkedData pub fn vortex_array::arrays::chunked::ChunkedData::clone(&self) -> vortex_array::arrays::chunked::ChunkedData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::chunked::ChunkedData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::chunked::ChunkedData pub fn vortex_array::arrays::chunked::ChunkedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::chunked::ChunkedData - -pub fn vortex_array::arrays::chunked::ChunkedData::into_array(self) -> vortex_array::ArrayRef - pub type vortex_array::arrays::chunked::ChunkedArray = vortex_array::Array pub mod vortex_array::arrays::constant @@ -1216,8 +1176,6 @@ impl vortex_array::VTable for vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ArrayData = vortex_array::arrays::constant::ConstantData -pub type vortex_array::arrays::Constant::Metadata = vortex_array::scalar::Scalar - pub type vortex_array::arrays::Constant::OperationsVTable = vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays::Constant @@ -1232,15 +1190,11 @@ pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Constant::dtype(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -1248,10 +1202,6 @@ pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::Array pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Constant::len(array: &vortex_array::arrays::constant::ConstantData) -> usize - -pub fn vortex_array::arrays::Constant::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Constant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1260,15 +1210,13 @@ pub fn vortex_array::arrays::Constant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Constant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -1308,15 +1256,9 @@ pub struct vortex_array::arrays::constant::ConstantData impl vortex_array::arrays::constant::ConstantData -pub fn vortex_array::arrays::constant::ConstantData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_array::arrays::constant::ConstantData::into_parts(self) -> vortex_array::scalar::Scalar -pub fn vortex_array::arrays::constant::ConstantData::is_empty(&self) -> bool - -pub fn vortex_array::arrays::constant::ConstantData::len(&self) -> usize - -pub fn vortex_array::arrays::constant::ConstantData::new(scalar: S, len: usize) -> Self where S: core::convert::Into +pub fn vortex_array::arrays::constant::ConstantData::new(scalar: S) -> Self where S: core::convert::Into pub fn vortex_array::arrays::constant::ConstantData::scalar(&self) -> &vortex_array::scalar::Scalar @@ -1324,18 +1266,10 @@ impl core::clone::Clone for vortex_array::arrays::constant::ConstantData pub fn vortex_array::arrays::constant::ConstantData::clone(&self) -> vortex_array::arrays::constant::ConstantData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::constant::ConstantData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::constant::ConstantData pub fn vortex_array::arrays::constant::ConstantData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::constant::ConstantData - -pub fn vortex_array::arrays::constant::ConstantData::into_array(self) -> vortex_array::ArrayRef - pub type vortex_array::arrays::constant::ConstantArray = vortex_array::Array pub mod vortex_array::arrays::datetime @@ -1422,8 +1356,6 @@ impl vortex_array::VTable for vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ArrayData = vortex_array::arrays::decimal::DecimalData -pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Decimal::OperationsVTable = vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::Decimal @@ -1438,15 +1370,11 @@ pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Decimal::dtype(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -1454,10 +1382,6 @@ pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayV pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Decimal::len(array: &vortex_array::arrays::decimal::DecimalData) -> usize - -pub fn vortex_array::arrays::Decimal::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Decimal::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1466,15 +1390,13 @@ pub fn vortex_array::arrays::Decimal::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -1532,7 +1454,7 @@ pub fn vortex_array::arrays::decimal::DecimalData::buffer_handle(&self) -> &vort pub fn vortex_array::arrays::decimal::DecimalData::decimal_dtype(&self) -> vortex_array::dtype::DecimalDType -pub fn vortex_array::arrays::decimal::DecimalData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::decimal::DecimalData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::decimal::DecimalData::from_iter>(iter: I, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self @@ -1572,18 +1494,10 @@ impl core::clone::Clone for vortex_array::arrays::decimal::DecimalData pub fn vortex_array::arrays::decimal::DecimalData::clone(&self) -> vortex_array::arrays::decimal::DecimalData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::decimal::DecimalData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::decimal::DecimalData pub fn vortex_array::arrays::decimal::DecimalData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::decimal::DecimalData - -pub fn vortex_array::arrays::decimal::DecimalData::into_array(self) -> vortex_array::ArrayRef - pub struct vortex_array::arrays::decimal::DecimalMaskedValidityRule impl core::default::Default for vortex_array::arrays::decimal::DecimalMaskedValidityRule @@ -1630,8 +1544,6 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData -pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -1646,15 +1558,11 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -1662,10 +1570,6 @@ pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::Arr pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize - -pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::dict::Dict::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1674,15 +1578,13 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -1746,8 +1648,6 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData -pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -1762,15 +1662,11 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -1778,10 +1674,6 @@ pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::Arr pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize - -pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::dict::Dict::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1790,15 +1682,13 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -1842,8 +1732,6 @@ pub struct vortex_array::arrays::dict::DictArrayParts pub vortex_array::arrays::dict::DictArrayParts::codes: vortex_array::ArrayRef -pub vortex_array::arrays::dict::DictArrayParts::dtype: vortex_array::dtype::DType - pub vortex_array::arrays::dict::DictArrayParts::values: vortex_array::ArrayRef pub struct vortex_array::arrays::dict::DictData @@ -1852,7 +1740,7 @@ impl vortex_array::arrays::dict::DictData pub fn vortex_array::arrays::dict::DictData::codes(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::dict::DictData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::dict::DictData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::dict::DictData::has_all_values_referenced(&self) -> bool @@ -1882,18 +1770,10 @@ impl core::clone::Clone for vortex_array::arrays::dict::DictData pub fn vortex_array::arrays::dict::DictData::clone(&self) -> vortex_array::arrays::dict::DictData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::dict::DictData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::dict::DictData pub fn vortex_array::arrays::dict::DictData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::dict::DictData - -pub fn vortex_array::arrays::dict::DictData::into_array(self) -> vortex_array::ArrayRef - pub struct vortex_array::arrays::dict::DictMetadata impl vortex_array::arrays::dict::DictMetadata @@ -2056,8 +1936,6 @@ impl vortex_array::VTable for vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData -pub type vortex_array::arrays::Extension::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -2072,15 +1950,11 @@ pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Extension::dtype(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -2088,10 +1962,6 @@ pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::Arra pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Extension::len(array: &vortex_array::arrays::extension::ExtensionData) -> usize - -pub fn vortex_array::arrays::Extension::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Extension::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2100,15 +1970,13 @@ pub fn vortex_array::arrays::Extension::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Extension::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Extension::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -2144,7 +2012,7 @@ pub struct vortex_array::arrays::extension::ExtensionData impl vortex_array::arrays::extension::ExtensionData -pub fn vortex_array::arrays::extension::ExtensionData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::extension::ExtensionData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::extension::ExtensionData::ext_dtype(&self) -> &vortex_array::dtype::extension::ExtDTypeRef @@ -2164,18 +2032,10 @@ impl core::clone::Clone for vortex_array::arrays::extension::ExtensionData pub fn vortex_array::arrays::extension::ExtensionData::clone(&self) -> vortex_array::arrays::extension::ExtensionData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::extension::ExtensionData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::extension::ExtensionData pub fn vortex_array::arrays::extension::ExtensionData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::into_array(self) -> vortex_array::ArrayRef - pub type vortex_array::arrays::extension::ExtensionArray = vortex_array::Array pub mod vortex_array::arrays::filter @@ -2202,8 +2062,6 @@ impl vortex_array::VTable for vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ArrayData = vortex_array::arrays::filter::FilterData -pub type vortex_array::arrays::Filter::Metadata = vortex_array::arrays::filter::vtable::FilterMetadata - pub type vortex_array::arrays::Filter::OperationsVTable = vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Filter @@ -2218,15 +2076,11 @@ pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::filter::vtable::FilterMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Filter::dtype(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -2234,10 +2088,6 @@ pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Filter::len(array: &vortex_array::arrays::filter::FilterData) -> usize - -pub fn vortex_array::arrays::Filter::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Filter::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2246,15 +2096,13 @@ pub fn vortex_array::arrays::Filter::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Filter::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Filter::vtable(_array: &vortex_array::arrays::filter::FilterData) -> &Self +pub fn vortex_array::arrays::Filter::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -2294,18 +2142,10 @@ impl core::clone::Clone for vortex_array::arrays::filter::FilterData pub fn vortex_array::arrays::filter::FilterData::clone(&self) -> vortex_array::arrays::filter::FilterData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::filter::FilterData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::filter::FilterData pub fn vortex_array::arrays::filter::FilterData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::filter::FilterData - -pub fn vortex_array::arrays::filter::FilterData::into_array(self) -> vortex_array::ArrayRef - pub struct vortex_array::arrays::filter::FilterExecuteAdaptor(pub V) impl core::default::Default for vortex_array::arrays::filter::FilterExecuteAdaptor @@ -2412,8 +2252,6 @@ impl vortex_array::VTable for vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ArrayData = vortex_array::arrays::fixed_size_list::FixedSizeListData -pub type vortex_array::arrays::FixedSizeList::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::FixedSizeList::OperationsVTable = vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arrays::FixedSizeList @@ -2428,15 +2266,11 @@ pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::FixedSizeList::dtype(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -2444,10 +2278,6 @@ pub fn vortex_array::arrays::FixedSizeList::execute_parent(array: vortex_array:: pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::FixedSizeList::len(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> usize - -pub fn vortex_array::arrays::FixedSizeList::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::FixedSizeList::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2456,15 +2286,13 @@ pub fn vortex_array::arrays::FixedSizeList::reduce(array: vortex_array::ArrayVie pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::FixedSizeList::vtable(_array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &Self +pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -2492,7 +2320,7 @@ pub struct vortex_array::arrays::fixed_size_list::FixedSizeListData impl vortex_array::arrays::fixed_size_list::FixedSizeListData -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::elements(&self) -> &vortex_array::ArrayRef @@ -2524,18 +2352,10 @@ impl core::clone::Clone for vortex_array::arrays::fixed_size_list::FixedSizeList pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::clone(&self) -> vortex_array::arrays::fixed_size_list::FixedSizeListData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::fixed_size_list::FixedSizeListData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::fixed_size_list::FixedSizeListData pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::fixed_size_list::FixedSizeListData - -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::into_array(self) -> vortex_array::ArrayRef - pub type vortex_array::arrays::fixed_size_list::FixedSizeListArray = vortex_array::Array pub mod vortex_array::arrays::list @@ -2562,8 +2382,6 @@ impl vortex_array::VTable for vortex_array::arrays::List pub type vortex_array::arrays::List::ArrayData = vortex_array::arrays::list::ListData -pub type vortex_array::arrays::List::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::List::OperationsVTable = vortex_array::arrays::List pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List @@ -2578,15 +2396,11 @@ pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::List::dtype(array: &vortex_array::arrays::list::ListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -2594,10 +2408,6 @@ pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::List::len(array: &vortex_array::arrays::list::ListData) -> usize - -pub fn vortex_array::arrays::List::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::List::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2606,15 +2416,13 @@ pub fn vortex_array::arrays::List::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::List::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::list::ListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::List::vtable(_array: &vortex_array::arrays::list::ListData) -> &Self +pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -2656,9 +2464,9 @@ pub struct vortex_array::arrays::list::ListData impl vortex_array::arrays::list::ListData -pub fn vortex_array::arrays::list::ListData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::list::ListData::dtype(&self) -> vortex_array::dtype::DType -pub fn vortex_array::arrays::list::ListData::element_dtype(&self) -> &alloc::sync::Arc +pub fn vortex_array::arrays::list::ListData::element_dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::list::ListData::elements(&self) -> &vortex_array::ArrayRef @@ -2696,18 +2504,10 @@ impl core::clone::Clone for vortex_array::arrays::list::ListData pub fn vortex_array::arrays::list::ListData::clone(&self) -> vortex_array::arrays::list::ListData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::list::ListData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::list::ListData pub fn vortex_array::arrays::list::ListData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::list::ListData - -pub fn vortex_array::arrays::list::ListData::into_array(self) -> vortex_array::ArrayRef - pub type vortex_array::arrays::list::ListArray = vortex_array::Array pub mod vortex_array::arrays::listview @@ -2744,8 +2544,6 @@ impl vortex_array::VTable for vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ArrayData = vortex_array::arrays::listview::ListViewData -pub type vortex_array::arrays::ListView::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::ListView::OperationsVTable = vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays::ListView @@ -2760,15 +2558,11 @@ pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::ListView::dtype(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -2776,10 +2570,6 @@ pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::Array pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::ListView::len(array: &vortex_array::arrays::listview::ListViewData) -> usize - -pub fn vortex_array::arrays::ListView::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::ListView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2788,15 +2578,13 @@ pub fn vortex_array::arrays::ListView::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::ListView::vtable(_array: &vortex_array::arrays::listview::ListViewData) -> &Self +pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -2836,7 +2624,7 @@ pub struct vortex_array::arrays::listview::ListViewData impl vortex_array::arrays::listview::ListViewData -pub fn vortex_array::arrays::listview::ListViewData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::listview::ListViewData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::listview::ListViewData::elements(&self) -> &vortex_array::ArrayRef @@ -2878,18 +2666,10 @@ impl core::clone::Clone for vortex_array::arrays::listview::ListViewData pub fn vortex_array::arrays::listview::ListViewData::clone(&self) -> vortex_array::arrays::listview::ListViewData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::listview::ListViewData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::listview::ListViewData pub fn vortex_array::arrays::listview::ListViewData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::listview::ListViewData - -pub fn vortex_array::arrays::listview::ListViewData::into_array(self) -> vortex_array::ArrayRef - pub fn vortex_array::arrays::listview::list_from_list_view(list_view: vortex_array::arrays::ListViewArray) -> vortex_error::VortexResult pub fn vortex_array::arrays::listview::list_view_from_list(list: vortex_array::arrays::ListArray, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -2922,8 +2702,6 @@ impl vortex_array::VTable for vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ArrayData = vortex_array::arrays::masked::MaskedData -pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked @@ -2938,15 +2716,11 @@ pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Masked::dtype(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -2954,10 +2728,6 @@ pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Masked::len(array: &vortex_array::arrays::masked::MaskedData) -> usize - -pub fn vortex_array::arrays::Masked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Masked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2966,15 +2736,13 @@ pub fn vortex_array::arrays::Masked::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Masked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Masked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3004,7 +2772,7 @@ impl vortex_array::arrays::masked::MaskedData pub fn vortex_array::arrays::masked::MaskedData::child(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::masked::MaskedData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::masked::MaskedData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::masked::MaskedData::is_empty(&self) -> bool @@ -3020,18 +2788,10 @@ impl core::clone::Clone for vortex_array::arrays::masked::MaskedData pub fn vortex_array::arrays::masked::MaskedData::clone(&self) -> vortex_array::arrays::masked::MaskedData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::masked::MaskedData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::masked::MaskedData pub fn vortex_array::arrays::masked::MaskedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::masked::MaskedData - -pub fn vortex_array::arrays::masked::MaskedData::into_array(self) -> vortex_array::ArrayRef - pub fn vortex_array::arrays::masked::mask_validity_canonical(canonical: vortex_array::Canonical, validity_mask: &vortex_mask::Mask, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub type vortex_array::arrays::masked::MaskedArray = vortex_array::Array @@ -3064,8 +2824,6 @@ impl vortex_array::VTable for vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ArrayData = vortex_array::arrays::null::NullData -pub type vortex_array::arrays::null::Null::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::null::Null::OperationsVTable = vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays::null::Null @@ -3080,15 +2838,11 @@ pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::null::Null::dtype(_array: &vortex_array::arrays::null::NullData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -3096,10 +2850,6 @@ pub fn vortex_array::arrays::null::Null::execute_parent(array: vortex_array::Arr pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::null::Null::len(array: &vortex_array::arrays::null::NullData) -> usize - -pub fn vortex_array::arrays::null::Null::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::null::Null::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3108,15 +2858,13 @@ pub fn vortex_array::arrays::null::Null::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::null::Null::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::null::Null::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3148,30 +2896,16 @@ pub struct vortex_array::arrays::null::NullData impl vortex_array::arrays::null::NullData -pub fn vortex_array::arrays::null::NullData::dtype(&self) -> &vortex_array::dtype::DType - -pub fn vortex_array::arrays::null::NullData::is_empty(&self) -> bool - -pub fn vortex_array::arrays::null::NullData::len(&self) -> usize - -pub fn vortex_array::arrays::null::NullData::new(len: usize) -> Self +pub fn vortex_array::arrays::null::NullData::new() -> Self impl core::clone::Clone for vortex_array::arrays::null::NullData pub fn vortex_array::arrays::null::NullData::clone(&self) -> vortex_array::arrays::null::NullData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::null::NullData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::null::NullData pub fn vortex_array::arrays::null::NullData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::null::NullData - -pub fn vortex_array::arrays::null::NullData::into_array(self) -> vortex_array::ArrayRef - pub type vortex_array::arrays::null::NullArray = vortex_array::Array pub mod vortex_array::arrays::patched @@ -3194,8 +2928,6 @@ impl vortex_array::VTable for vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray -pub type vortex_array::arrays::patched::Patched::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -3210,15 +2942,11 @@ pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::patched::Patched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -3226,10 +2954,6 @@ pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_arra pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::patched::Patched::len(array: &Self::ArrayData) -> usize - -pub fn vortex_array::arrays::patched::Patched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::patched::Patched::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3238,15 +2962,13 @@ pub fn vortex_array::arrays::patched::Patched::reduce(array: vortex_array::Array pub fn vortex_array::arrays::patched::Patched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::patched::Patched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::patched::Patched::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3278,17 +3000,21 @@ pub fn vortex_array::arrays::patched::PatchedArray::base_array(&self) -> &vortex pub fn vortex_array::arrays::patched::PatchedArray::lane_offsets(&self) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::PatchedArray::len(&self) -> usize + pub fn vortex_array::arrays::patched::PatchedArray::patch_indices(&self) -> &vortex_array::ArrayRef pub fn vortex_array::arrays::patched::PatchedArray::patch_values(&self) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::PatchedArray::validate_against_outer(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> + impl vortex_array::arrays::patched::PatchedArray pub fn vortex_array::arrays::patched::PatchedArray::from_array_and_patches(inner: vortex_array::ArrayRef, patches: &vortex_array::patches::Patches, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::arrays::patched::PatchedArray -pub fn vortex_array::arrays::patched::PatchedArray::to_array(&self) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::PatchedArray::to_array(&self) -> vortex_array::ArrayRef where Self: core::clone::Clone + vortex_array::IntoArray impl core::clone::Clone for vortex_array::arrays::patched::PatchedArray @@ -3296,7 +3022,7 @@ pub fn vortex_array::arrays::patched::PatchedArray::clone(&self) -> vortex_array impl core::convert::From for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> vortex_array::ArrayRef +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> Self impl core::fmt::Debug for vortex_array::arrays::patched::PatchedArray @@ -3416,8 +3142,6 @@ impl vortex_array::VTable for vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ArrayData = vortex_array::arrays::primitive::PrimitiveData -pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive @@ -3432,15 +3156,11 @@ pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Primitive::dtype(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -3448,10 +3168,6 @@ pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::Arra pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Primitive::len(array: &vortex_array::arrays::primitive::PrimitiveData) -> usize - -pub fn vortex_array::arrays::Primitive::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Primitive::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3460,15 +3176,13 @@ pub fn vortex_array::arrays::Primitive::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3526,7 +3240,7 @@ impl vortex_array::arrays::primitive::PrimitiveData pub fn vortex_array::arrays::primitive::PrimitiveData::buffer_handle(&self) -> &vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::primitive::PrimitiveData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::primitive::PrimitiveData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::primitive::PrimitiveData::from_buffer_handle(handle: vortex_array::buffer::BufferHandle, ptype: vortex_array::dtype::PType, validity: vortex_array::validity::Validity) -> Self @@ -3578,18 +3292,10 @@ impl core::clone::Clone for vortex_array::arrays::primitive::PrimitiveData pub fn vortex_array::arrays::primitive::PrimitiveData::clone(&self) -> vortex_array::arrays::primitive::PrimitiveData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::primitive::PrimitiveData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::primitive::PrimitiveData pub fn vortex_array::arrays::primitive::PrimitiveData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::primitive::PrimitiveData - -pub fn vortex_array::arrays::primitive::PrimitiveData::into_array(self) -> vortex_array::ArrayRef - pub struct vortex_array::arrays::primitive::PrimitiveMaskedValidityRule impl core::default::Default for vortex_array::arrays::primitive::PrimitiveMaskedValidityRule @@ -3664,16 +3370,10 @@ impl vortex_array::arrays::scalar_fn::ScalarFnData pub fn vortex_array::arrays::scalar_fn::ScalarFnData::children(&self) -> alloc::vec::Vec -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_array::arrays::scalar_fn::ScalarFnData::get_child(&self, idx: usize) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::is_empty(&self) -> bool - pub fn vortex_array::arrays::scalar_fn::ScalarFnData::iter_children(&self) -> impl core::iter::traits::iterator::Iterator + '_ -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::len(&self) -> usize - pub fn vortex_array::arrays::scalar_fn::ScalarFnData::nchildren(&self) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnData::scalar_fn(&self) -> &vortex_array::scalar_fn::ScalarFnRef @@ -3684,18 +3384,10 @@ impl core::clone::Clone for vortex_array::arrays::scalar_fn::ScalarFnData pub fn vortex_array::arrays::scalar_fn::ScalarFnData::clone(&self) -> vortex_array::arrays::scalar_fn::ScalarFnData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::scalar_fn::ScalarFnData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::scalar_fn::ScalarFnData pub fn vortex_array::arrays::scalar_fn::ScalarFnData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::scalar_fn::ScalarFnData - -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::into_array(self) -> vortex_array::ArrayRef - pub struct vortex_array::arrays::scalar_fn::ScalarFnVTable impl core::clone::Clone for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -3714,8 +3406,6 @@ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::Metadata = vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata - pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -3730,15 +3420,11 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::dtype(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -3746,10 +3432,6 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vo pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::len(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> usize - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3758,15 +3440,13 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_arr pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::vtable(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &Self +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3808,8 +3488,6 @@ impl vortex_array::VTable for vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ArrayData = vortex_array::arrays::shared::SharedData -pub type vortex_array::arrays::Shared::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Shared::OperationsVTable = vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Shared @@ -3824,15 +3502,11 @@ pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Shared::dtype(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -3840,10 +3514,6 @@ pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Shared::len(array: &vortex_array::arrays::shared::SharedData) -> usize - -pub fn vortex_array::arrays::Shared::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Shared::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3852,15 +3522,13 @@ pub fn vortex_array::arrays::Shared::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Shared::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Shared::vtable(_array: &vortex_array::arrays::shared::SharedData) -> &Self +pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3888,18 +3556,10 @@ impl core::clone::Clone for vortex_array::arrays::shared::SharedData pub fn vortex_array::arrays::shared::SharedData::clone(&self) -> vortex_array::arrays::shared::SharedData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::shared::SharedData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::shared::SharedData pub fn vortex_array::arrays::shared::SharedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::shared::SharedData - -pub fn vortex_array::arrays::shared::SharedData::into_array(self) -> vortex_array::ArrayRef - pub type vortex_array::arrays::shared::SharedArray = vortex_array::Array pub mod vortex_array::arrays::slice @@ -3926,8 +3586,6 @@ impl vortex_array::VTable for vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ArrayData = vortex_array::arrays::slice::SliceData -pub type vortex_array::arrays::slice::Slice::Metadata = vortex_array::arrays::slice::SliceMetadata - pub type vortex_array::arrays::slice::Slice::OperationsVTable = vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arrays::slice::Slice @@ -3942,15 +3600,11 @@ pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::slice::SliceMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -3958,10 +3612,6 @@ pub fn vortex_array::arrays::slice::Slice::execute_parent(array: vortex_array::A pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::slice::Slice::len(array: &vortex_array::arrays::slice::SliceData) -> usize - -pub fn vortex_array::arrays::slice::Slice::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::slice::Slice::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3970,15 +3620,13 @@ pub fn vortex_array::arrays::slice::Slice::reduce(array: vortex_array::ArrayView pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::slice::Slice::vtable(_array: &vortex_array::arrays::slice::SliceData) -> &Self +pub fn vortex_array::arrays::slice::Slice::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -4022,18 +3670,10 @@ impl core::clone::Clone for vortex_array::arrays::slice::SliceData pub fn vortex_array::arrays::slice::SliceData::clone(&self) -> vortex_array::arrays::slice::SliceData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::slice::SliceData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::slice::SliceData pub fn vortex_array::arrays::slice::SliceData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::slice::SliceData - -pub fn vortex_array::arrays::slice::SliceData::into_array(self) -> vortex_array::ArrayRef - pub struct vortex_array::arrays::slice::SliceExecuteAdaptor(pub V) impl core::default::Default for vortex_array::arrays::slice::SliceExecuteAdaptor @@ -4174,8 +3814,6 @@ impl vortex_array::VTable for vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ArrayData = vortex_array::arrays::struct_::StructData -pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct @@ -4190,15 +3828,11 @@ pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Struct::dtype(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -4206,10 +3840,6 @@ pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Struct::len(array: &vortex_array::arrays::struct_::StructData) -> usize - -pub fn vortex_array::arrays::Struct::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Struct::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -4218,15 +3848,13 @@ pub fn vortex_array::arrays::Struct::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Struct::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Struct::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -4266,7 +3894,7 @@ pub struct vortex_array::arrays::struct_::StructData impl vortex_array::arrays::struct_::StructData -pub fn vortex_array::arrays::struct_::StructData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::struct_::StructData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::struct_::StructData::from_fields>(items: &[(N, vortex_array::ArrayRef)]) -> vortex_error::VortexResult @@ -4292,7 +3920,7 @@ pub fn vortex_array::arrays::struct_::StructData::project(&self, projection: &[v pub fn vortex_array::arrays::struct_::StructData::remove_column(&mut self, name: impl core::convert::Into) -> core::option::Option -pub fn vortex_array::arrays::struct_::StructData::struct_fields(&self) -> &vortex_array::dtype::StructFields +pub fn vortex_array::arrays::struct_::StructData::struct_fields(&self) -> vortex_array::dtype::StructFields pub fn vortex_array::arrays::struct_::StructData::try_from_iter, A: vortex_array::IntoArray, T: core::iter::traits::collect::IntoIterator>(iter: T) -> vortex_error::VortexResult @@ -4312,6 +3940,8 @@ pub fn vortex_array::arrays::struct_::StructData::unmasked_fields(&self) -> allo pub fn vortex_array::arrays::struct_::StructData::validate(fields: &[vortex_array::ArrayRef], dtype: &vortex_array::dtype::StructFields, length: usize, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::struct_::StructData::validate_against_outer(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::struct_::StructData::validity(&self) -> vortex_array::validity::Validity impl vortex_array::arrays::struct_::StructData @@ -4322,18 +3952,10 @@ impl core::clone::Clone for vortex_array::arrays::struct_::StructData pub fn vortex_array::arrays::struct_::StructData::clone(&self) -> vortex_array::arrays::struct_::StructData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::struct_::StructData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::struct_::StructData pub fn vortex_array::arrays::struct_::StructData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::struct_::StructData - -pub fn vortex_array::arrays::struct_::StructData::into_array(self) -> vortex_array::ArrayRef - pub type vortex_array::arrays::struct_::StructArray = vortex_array::Array pub mod vortex_array::arrays::varbin @@ -4390,8 +4012,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ArrayData = vortex_array::arrays::varbin::VarBinData -pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::VarBin::OperationsVTable = vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::VarBin @@ -4406,15 +4026,11 @@ pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBin::dtype(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -4422,10 +4038,6 @@ pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::VarBin::len(array: &vortex_array::arrays::varbin::VarBinData) -> usize - -pub fn vortex_array::arrays::VarBin::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBin::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -4434,15 +4046,13 @@ pub fn vortex_array::arrays::VarBin::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBin::vtable(_array: &vortex_array::arrays::varbin::VarBinData) -> &Self +pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -4482,7 +4092,7 @@ pub fn vortex_array::arrays::varbin::VarBinData::bytes(&self) -> &vortex_buffer: pub fn vortex_array::arrays::varbin::VarBinData::bytes_handle(&self) -> &vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::varbin::VarBinData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::varbin::VarBinData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::varbin::VarBinData::from_iter, I: core::iter::traits::collect::IntoIterator>>(iter: I, dtype: vortex_array::dtype::DType) -> Self @@ -4524,6 +4134,8 @@ pub fn vortex_array::arrays::varbin::VarBinData::into_parts(self) -> (vortex_arr pub fn vortex_array::arrays::varbin::VarBinData::offset_at(&self, index: usize) -> usize +pub fn vortex_array::arrays::varbin::VarBinData::validate_against_outer(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> + impl core::clone::Clone for vortex_array::arrays::varbin::VarBinData pub fn vortex_array::arrays::varbin::VarBinData::clone(&self) -> vortex_array::arrays::varbin::VarBinData @@ -4560,10 +4172,6 @@ impl core::convert::From>>) -> Self -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::varbin::VarBinData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::varbin::VarBinData pub fn vortex_array::arrays::varbin::VarBinData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result @@ -4576,10 +4184,6 @@ impl core::iter::traits::collect::FromIterator>>>(iter: T) -> Self -impl vortex_array::IntoArray for vortex_array::arrays::varbin::VarBinData - -pub fn vortex_array::arrays::varbin::VarBinData::into_array(self) -> vortex_array::ArrayRef - impl<'a> core::iter::traits::collect::FromIterator> for vortex_array::arrays::varbin::VarBinData pub fn vortex_array::arrays::varbin::VarBinData::from_iter>>(iter: T) -> Self @@ -4798,8 +4402,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ArrayData = vortex_array::arrays::varbinview::VarBinViewData -pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView @@ -4814,15 +4416,11 @@ pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBinView::dtype(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -4830,10 +4428,6 @@ pub fn vortex_array::arrays::VarBinView::execute_parent(array: vortex_array::Arr pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::VarBinView::len(array: &vortex_array::arrays::varbinview::VarBinViewData) -> usize - -pub fn vortex_array::arrays::VarBinView::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBinView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -4842,15 +4436,13 @@ pub fn vortex_array::arrays::VarBinView::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBinView::vtable(_array: &vortex_array::arrays::varbinview::VarBinViewData) -> &Self +pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -4898,7 +4490,7 @@ pub fn vortex_array::arrays::varbinview::VarBinViewData::bytes_at(&self, index: pub fn vortex_array::arrays::varbinview::VarBinViewData::data_buffers(&self) -> &alloc::sync::Arc<[vortex_array::buffer::BufferHandle]> -pub fn vortex_array::arrays::varbinview::VarBinViewData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::varbinview::VarBinViewData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter, I: core::iter::traits::collect::IntoIterator>>(iter: I, dtype: vortex_array::dtype::DType) -> Self @@ -4930,6 +4522,8 @@ pub fn vortex_array::arrays::varbinview::VarBinViewData::try_new_handle(views: v pub fn vortex_array::arrays::varbinview::VarBinViewData::validate(views: &vortex_buffer::buffer::Buffer, buffers: &alloc::sync::Arc<[vortex_buffer::ByteBuffer]>, dtype: &vortex_array::dtype::DType, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::varbinview::VarBinViewData::validate_against_outer(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> + pub fn vortex_array::arrays::varbinview::VarBinViewData::validity(&self) -> vortex_array::validity::Validity pub fn vortex_array::arrays::varbinview::VarBinViewData::validity_mask(&self) -> vortex_mask::Mask @@ -4942,10 +4536,6 @@ impl core::clone::Clone for vortex_array::arrays::varbinview::VarBinViewData pub fn vortex_array::arrays::varbinview::VarBinViewData::clone(&self) -> vortex_array::arrays::varbinview::VarBinViewData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::varbinview::VarBinViewData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::varbinview::VarBinViewData pub fn vortex_array::arrays::varbinview::VarBinViewData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result @@ -4958,10 +4548,6 @@ impl core::iter::traits::collect::FromIterator>>>(iter: T) -> Self -impl vortex_array::IntoArray for vortex_array::arrays::varbinview::VarBinViewData - -pub fn vortex_array::arrays::varbinview::VarBinViewData::into_array(self) -> vortex_array::ArrayRef - impl<'a> core::iter::traits::collect::FromIterator> for vortex_array::arrays::varbinview::VarBinViewData pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter>>(iter: T) -> Self @@ -4996,8 +4582,6 @@ impl vortex_array::VTable for vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ArrayData = vortex_array::arrays::variant::VariantData -pub type vortex_array::arrays::Variant::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Variant::OperationsVTable = vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::Variant @@ -5012,15 +4596,11 @@ pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Variant::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5028,10 +4608,6 @@ pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayV pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Variant::len(array: &Self::ArrayData) -> usize - -pub fn vortex_array::arrays::Variant::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Variant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5040,15 +4616,13 @@ pub fn vortex_array::arrays::Variant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Variant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Variant::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5074,18 +4648,10 @@ impl core::clone::Clone for vortex_array::arrays::variant::VariantData pub fn vortex_array::arrays::variant::VariantData::clone(&self) -> vortex_array::arrays::variant::VariantData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::variant::VariantData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::variant::VariantData pub fn vortex_array::arrays::variant::VariantData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::variant::VariantData - -pub fn vortex_array::arrays::variant::VariantData::into_array(self) -> vortex_array::ArrayRef - pub type vortex_array::arrays::variant::VariantArray = vortex_array::Array pub struct vortex_array::arrays::Bool @@ -5110,8 +4676,6 @@ impl vortex_array::VTable for vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ArrayData = vortex_array::arrays::bool::BoolData -pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Bool::OperationsVTable = vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool @@ -5126,15 +4690,11 @@ pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Bool::dtype(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5142,10 +4702,6 @@ pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Bool::len(array: &vortex_array::arrays::bool::BoolData) -> usize - -pub fn vortex_array::arrays::Bool::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Bool::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5154,15 +4710,13 @@ pub fn vortex_array::arrays::Bool::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Bool::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5222,8 +4776,6 @@ impl vortex_array::VTable for vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ArrayData = vortex_array::arrays::chunked::ChunkedData -pub type vortex_array::arrays::Chunked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Chunked::OperationsVTable = vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::Chunked @@ -5238,15 +4790,11 @@ pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Chunked::dtype(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5254,10 +4802,6 @@ pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayV pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Chunked::len(array: &vortex_array::arrays::chunked::ChunkedData) -> usize - -pub fn vortex_array::arrays::Chunked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Chunked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5266,15 +4810,13 @@ pub fn vortex_array::arrays::Chunked::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Chunked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5336,8 +4878,6 @@ impl vortex_array::VTable for vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ArrayData = vortex_array::arrays::constant::ConstantData -pub type vortex_array::arrays::Constant::Metadata = vortex_array::scalar::Scalar - pub type vortex_array::arrays::Constant::OperationsVTable = vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays::Constant @@ -5352,15 +4892,11 @@ pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Constant::dtype(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5368,10 +4904,6 @@ pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::Array pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Constant::len(array: &vortex_array::arrays::constant::ConstantData) -> usize - -pub fn vortex_array::arrays::Constant::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Constant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5380,15 +4912,13 @@ pub fn vortex_array::arrays::Constant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Constant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5446,8 +4976,6 @@ impl vortex_array::VTable for vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ArrayData = vortex_array::arrays::decimal::DecimalData -pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Decimal::OperationsVTable = vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::Decimal @@ -5462,15 +4990,11 @@ pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Decimal::dtype(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5478,10 +5002,6 @@ pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayV pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Decimal::len(array: &vortex_array::arrays::decimal::DecimalData) -> usize - -pub fn vortex_array::arrays::Decimal::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Decimal::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5490,15 +5010,13 @@ pub fn vortex_array::arrays::Decimal::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5558,8 +5076,6 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData -pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -5574,15 +5090,11 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5590,10 +5102,6 @@ pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::Arr pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize - -pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::dict::Dict::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5602,15 +5110,13 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5672,8 +5178,6 @@ impl vortex_array::VTable for vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData -pub type vortex_array::arrays::Extension::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -5688,15 +5192,11 @@ pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Extension::dtype(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5704,10 +5204,6 @@ pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::Arra pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Extension::len(array: &vortex_array::arrays::extension::ExtensionData) -> usize - -pub fn vortex_array::arrays::Extension::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Extension::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5716,15 +5212,13 @@ pub fn vortex_array::arrays::Extension::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Extension::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Extension::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5778,8 +5272,6 @@ impl vortex_array::VTable for vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ArrayData = vortex_array::arrays::filter::FilterData -pub type vortex_array::arrays::Filter::Metadata = vortex_array::arrays::filter::vtable::FilterMetadata - pub type vortex_array::arrays::Filter::OperationsVTable = vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Filter @@ -5794,15 +5286,11 @@ pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::filter::vtable::FilterMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Filter::dtype(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5810,10 +5298,6 @@ pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Filter::len(array: &vortex_array::arrays::filter::FilterData) -> usize - -pub fn vortex_array::arrays::Filter::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Filter::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5822,15 +5306,13 @@ pub fn vortex_array::arrays::Filter::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Filter::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Filter::vtable(_array: &vortex_array::arrays::filter::FilterData) -> &Self +pub fn vortex_array::arrays::Filter::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5860,8 +5342,6 @@ impl vortex_array::VTable for vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ArrayData = vortex_array::arrays::fixed_size_list::FixedSizeListData -pub type vortex_array::arrays::FixedSizeList::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::FixedSizeList::OperationsVTable = vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arrays::FixedSizeList @@ -5876,15 +5356,11 @@ pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::FixedSizeList::dtype(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5892,10 +5368,6 @@ pub fn vortex_array::arrays::FixedSizeList::execute_parent(array: vortex_array:: pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::FixedSizeList::len(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> usize - -pub fn vortex_array::arrays::FixedSizeList::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::FixedSizeList::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5904,15 +5376,13 @@ pub fn vortex_array::arrays::FixedSizeList::reduce(array: vortex_array::ArrayVie pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::FixedSizeList::vtable(_array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &Self +pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5958,8 +5428,6 @@ impl vortex_array::VTable for vortex_array::arrays::List pub type vortex_array::arrays::List::ArrayData = vortex_array::arrays::list::ListData -pub type vortex_array::arrays::List::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::List::OperationsVTable = vortex_array::arrays::List pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List @@ -5974,15 +5442,11 @@ pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::List::dtype(array: &vortex_array::arrays::list::ListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5990,10 +5454,6 @@ pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::List::len(array: &vortex_array::arrays::list::ListData) -> usize - -pub fn vortex_array::arrays::List::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::List::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6002,15 +5462,13 @@ pub fn vortex_array::arrays::List::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::List::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::list::ListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::List::vtable(_array: &vortex_array::arrays::list::ListData) -> &Self +pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6060,8 +5518,6 @@ impl vortex_array::VTable for vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ArrayData = vortex_array::arrays::listview::ListViewData -pub type vortex_array::arrays::ListView::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::ListView::OperationsVTable = vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays::ListView @@ -6076,15 +5532,11 @@ pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::ListView::dtype(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6092,10 +5544,6 @@ pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::Array pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::ListView::len(array: &vortex_array::arrays::listview::ListViewData) -> usize - -pub fn vortex_array::arrays::ListView::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::ListView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6104,15 +5552,13 @@ pub fn vortex_array::arrays::ListView::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::ListView::vtable(_array: &vortex_array::arrays::listview::ListViewData) -> &Self +pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6158,8 +5604,6 @@ impl vortex_array::VTable for vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ArrayData = vortex_array::arrays::masked::MaskedData -pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked @@ -6174,15 +5618,11 @@ pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Masked::dtype(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6190,10 +5630,6 @@ pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Masked::len(array: &vortex_array::arrays::masked::MaskedData) -> usize - -pub fn vortex_array::arrays::Masked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Masked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6202,15 +5638,13 @@ pub fn vortex_array::arrays::Masked::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Masked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Masked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6260,8 +5694,6 @@ impl vortex_array::VTable for vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ArrayData = vortex_array::arrays::null::NullData -pub type vortex_array::arrays::null::Null::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::null::Null::OperationsVTable = vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays::null::Null @@ -6276,15 +5708,11 @@ pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::null::Null::dtype(_array: &vortex_array::arrays::null::NullData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6292,10 +5720,6 @@ pub fn vortex_array::arrays::null::Null::execute_parent(array: vortex_array::Arr pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::null::Null::len(array: &vortex_array::arrays::null::NullData) -> usize - -pub fn vortex_array::arrays::null::Null::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::null::Null::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6304,15 +5728,13 @@ pub fn vortex_array::arrays::null::Null::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::null::Null::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::null::Null::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6358,8 +5780,6 @@ impl vortex_array::VTable for vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray -pub type vortex_array::arrays::patched::Patched::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -6374,15 +5794,11 @@ pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::patched::Patched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6390,10 +5806,6 @@ pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_arra pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::patched::Patched::len(array: &Self::ArrayData) -> usize - -pub fn vortex_array::arrays::patched::Patched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::patched::Patched::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6402,15 +5814,13 @@ pub fn vortex_array::arrays::patched::Patched::reduce(array: vortex_array::Array pub fn vortex_array::arrays::patched::Patched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::patched::Patched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::patched::Patched::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6442,17 +5852,21 @@ pub fn vortex_array::arrays::patched::PatchedArray::base_array(&self) -> &vortex pub fn vortex_array::arrays::patched::PatchedArray::lane_offsets(&self) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::PatchedArray::len(&self) -> usize + pub fn vortex_array::arrays::patched::PatchedArray::patch_indices(&self) -> &vortex_array::ArrayRef pub fn vortex_array::arrays::patched::PatchedArray::patch_values(&self) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::PatchedArray::validate_against_outer(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> + impl vortex_array::arrays::patched::PatchedArray pub fn vortex_array::arrays::patched::PatchedArray::from_array_and_patches(inner: vortex_array::ArrayRef, patches: &vortex_array::patches::Patches, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::arrays::patched::PatchedArray -pub fn vortex_array::arrays::patched::PatchedArray::to_array(&self) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::PatchedArray::to_array(&self) -> vortex_array::ArrayRef where Self: core::clone::Clone + vortex_array::IntoArray impl core::clone::Clone for vortex_array::arrays::patched::PatchedArray @@ -6460,7 +5874,7 @@ pub fn vortex_array::arrays::patched::PatchedArray::clone(&self) -> vortex_array impl core::convert::From for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> vortex_array::ArrayRef +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> Self impl core::fmt::Debug for vortex_array::arrays::patched::PatchedArray @@ -6492,8 +5906,6 @@ impl vortex_array::VTable for vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ArrayData = vortex_array::arrays::primitive::PrimitiveData -pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive @@ -6508,15 +5920,11 @@ pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Primitive::dtype(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6524,10 +5932,6 @@ pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::Arra pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Primitive::len(array: &vortex_array::arrays::primitive::PrimitiveData) -> usize - -pub fn vortex_array::arrays::Primitive::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Primitive::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6536,15 +5940,13 @@ pub fn vortex_array::arrays::Primitive::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6600,8 +6002,6 @@ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::Metadata = vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata - pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -6616,15 +6016,11 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::dtype(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6632,10 +6028,6 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vo pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::len(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> usize - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6644,15 +6036,13 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_arr pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::vtable(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &Self +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6682,8 +6072,6 @@ impl vortex_array::VTable for vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ArrayData = vortex_array::arrays::shared::SharedData -pub type vortex_array::arrays::Shared::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Shared::OperationsVTable = vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Shared @@ -6698,15 +6086,11 @@ pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Shared::dtype(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6714,10 +6098,6 @@ pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Shared::len(array: &vortex_array::arrays::shared::SharedData) -> usize - -pub fn vortex_array::arrays::Shared::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Shared::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6726,15 +6106,13 @@ pub fn vortex_array::arrays::Shared::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Shared::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Shared::vtable(_array: &vortex_array::arrays::shared::SharedData) -> &Self +pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6764,8 +6142,6 @@ impl vortex_array::VTable for vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ArrayData = vortex_array::arrays::slice::SliceData -pub type vortex_array::arrays::slice::Slice::Metadata = vortex_array::arrays::slice::SliceMetadata - pub type vortex_array::arrays::slice::Slice::OperationsVTable = vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arrays::slice::Slice @@ -6780,15 +6156,11 @@ pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::slice::SliceMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6796,10 +6168,6 @@ pub fn vortex_array::arrays::slice::Slice::execute_parent(array: vortex_array::A pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::slice::Slice::len(array: &vortex_array::arrays::slice::SliceData) -> usize - -pub fn vortex_array::arrays::slice::Slice::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::slice::Slice::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6808,15 +6176,13 @@ pub fn vortex_array::arrays::slice::Slice::reduce(array: vortex_array::ArrayView pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::slice::Slice::vtable(_array: &vortex_array::arrays::slice::SliceData) -> &Self +pub fn vortex_array::arrays::slice::Slice::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6850,8 +6216,6 @@ impl vortex_array::VTable for vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ArrayData = vortex_array::arrays::struct_::StructData -pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct @@ -6866,15 +6230,11 @@ pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Struct::dtype(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6882,10 +6242,6 @@ pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Struct::len(array: &vortex_array::arrays::struct_::StructData) -> usize - -pub fn vortex_array::arrays::Struct::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Struct::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6894,15 +6250,13 @@ pub fn vortex_array::arrays::Struct::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Struct::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Struct::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6956,8 +6310,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ArrayData = vortex_array::arrays::varbin::VarBinData -pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::VarBin::OperationsVTable = vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::VarBin @@ -6972,15 +6324,11 @@ pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBin::dtype(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6988,10 +6336,6 @@ pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::VarBin::len(array: &vortex_array::arrays::varbin::VarBinData) -> usize - -pub fn vortex_array::arrays::VarBin::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBin::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -7000,15 +6344,13 @@ pub fn vortex_array::arrays::VarBin::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBin::vtable(_array: &vortex_array::arrays::varbin::VarBinData) -> &Self +pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -7062,8 +6404,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ArrayData = vortex_array::arrays::varbinview::VarBinViewData -pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView @@ -7078,15 +6418,11 @@ pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBinView::dtype(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -7094,10 +6430,6 @@ pub fn vortex_array::arrays::VarBinView::execute_parent(array: vortex_array::Arr pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::VarBinView::len(array: &vortex_array::arrays::varbinview::VarBinViewData) -> usize - -pub fn vortex_array::arrays::VarBinView::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBinView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -7106,15 +6438,13 @@ pub fn vortex_array::arrays::VarBinView::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBinView::vtable(_array: &vortex_array::arrays::varbinview::VarBinViewData) -> &Self +pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -7164,8 +6494,6 @@ impl vortex_array::VTable for vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ArrayData = vortex_array::arrays::variant::VariantData -pub type vortex_array::arrays::Variant::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Variant::OperationsVTable = vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::Variant @@ -7180,15 +6508,11 @@ pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Variant::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -7196,10 +6520,6 @@ pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayV pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Variant::len(array: &Self::ArrayData) -> usize - -pub fn vortex_array::arrays::Variant::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Variant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -7208,15 +6528,13 @@ pub fn vortex_array::arrays::Variant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Variant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Variant::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -7570,11 +6888,11 @@ impl core::clone::Clone for vortex_array::buffer::BufferHandle pub fn vortex_array::buffer::BufferHandle::clone(&self) -> vortex_array::buffer::BufferHandle -impl core::convert::TryFrom for vortex_array::serde::ArrayParts +impl core::convert::TryFrom for vortex_array::serde::SerializedArray -pub type vortex_array::serde::ArrayParts::Error = vortex_error::VortexError +pub type vortex_array::serde::SerializedArray::Error = vortex_error::VortexError -pub fn vortex_array::serde::ArrayParts::try_from(value: vortex_array::buffer::BufferHandle) -> core::result::Result +pub fn vortex_array::serde::SerializedArray::try_from(value: vortex_array::buffer::BufferHandle) -> core::result::Result impl core::fmt::Debug for vortex_array::buffer::BufferHandle @@ -18804,67 +18122,67 @@ pub fn vortex_array::serde::ArrayNodeFlatBuffer<'a>::try_new(ctx: &'a vortex_arr pub fn vortex_array::serde::ArrayNodeFlatBuffer<'a>::try_write_flatbuffer<'fb>(&self, fbb: &mut flatbuffers::builder::FlatBufferBuilder<'fb>) -> vortex_error::VortexResult>> -pub struct vortex_array::serde::ArrayParts +pub struct vortex_array::serde::SerializeOptions -impl vortex_array::serde::ArrayParts +pub vortex_array::serde::SerializeOptions::include_padding: bool -pub fn vortex_array::serde::ArrayParts::buffer(&self, idx: usize) -> vortex_error::VortexResult +pub vortex_array::serde::SerializeOptions::offset: usize -pub fn vortex_array::serde::ArrayParts::buffer_lengths(&self) -> alloc::vec::Vec +impl core::default::Default for vortex_array::serde::SerializeOptions -pub fn vortex_array::serde::ArrayParts::child(&self, idx: usize) -> vortex_array::serde::ArrayParts +pub fn vortex_array::serde::SerializeOptions::default() -> vortex_array::serde::SerializeOptions -pub fn vortex_array::serde::ArrayParts::decode(&self, dtype: &vortex_array::dtype::DType, len: usize, ctx: &vortex_session::registry::ReadContext, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +impl core::fmt::Debug for vortex_array::serde::SerializeOptions -pub fn vortex_array::serde::ArrayParts::encoding_id(&self) -> u16 +pub fn vortex_array::serde::SerializeOptions::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::serde::ArrayParts::from_array_tree(array_tree: impl core::convert::Into) -> vortex_error::VortexResult +pub struct vortex_array::serde::SerializedArray -pub fn vortex_array::serde::ArrayParts::from_flatbuffer_and_segment(array_tree: vortex_buffer::ByteBuffer, segment: vortex_array::buffer::BufferHandle) -> vortex_error::VortexResult +impl vortex_array::serde::SerializedArray -pub fn vortex_array::serde::ArrayParts::from_flatbuffer_and_segment_with_overrides(array_tree: vortex_buffer::ByteBuffer, segment: vortex_array::buffer::BufferHandle, buffer_overrides: &vortex_utils::aliases::hash_map::HashMap) -> vortex_error::VortexResult +pub fn vortex_array::serde::SerializedArray::buffer(&self, idx: usize) -> vortex_error::VortexResult -pub fn vortex_array::serde::ArrayParts::from_flatbuffer_with_buffers(array_tree: impl core::convert::Into, buffers: alloc::vec::Vec) -> vortex_error::VortexResult +pub fn vortex_array::serde::SerializedArray::buffer_lengths(&self) -> alloc::vec::Vec -pub fn vortex_array::serde::ArrayParts::metadata(&self) -> &[u8] +pub fn vortex_array::serde::SerializedArray::child(&self, idx: usize) -> vortex_array::serde::SerializedArray -pub fn vortex_array::serde::ArrayParts::nbuffers(&self) -> usize +pub fn vortex_array::serde::SerializedArray::decode(&self, dtype: &vortex_array::dtype::DType, len: usize, ctx: &vortex_session::registry::ReadContext, session: &vortex_session::VortexSession) -> vortex_error::VortexResult -pub fn vortex_array::serde::ArrayParts::nchildren(&self) -> usize +pub fn vortex_array::serde::SerializedArray::encoding_id(&self) -> u16 -impl core::clone::Clone for vortex_array::serde::ArrayParts +pub fn vortex_array::serde::SerializedArray::from_array_tree(array_tree: impl core::convert::Into) -> vortex_error::VortexResult -pub fn vortex_array::serde::ArrayParts::clone(&self) -> vortex_array::serde::ArrayParts +pub fn vortex_array::serde::SerializedArray::from_flatbuffer_and_segment(array_tree: vortex_buffer::ByteBuffer, segment: vortex_array::buffer::BufferHandle) -> vortex_error::VortexResult -impl core::convert::TryFrom for vortex_array::serde::ArrayParts +pub fn vortex_array::serde::SerializedArray::from_flatbuffer_and_segment_with_overrides(array_tree: vortex_buffer::ByteBuffer, segment: vortex_array::buffer::BufferHandle, buffer_overrides: &vortex_utils::aliases::hash_map::HashMap) -> vortex_error::VortexResult -pub type vortex_array::serde::ArrayParts::Error = vortex_error::VortexError +pub fn vortex_array::serde::SerializedArray::from_flatbuffer_with_buffers(array_tree: impl core::convert::Into, buffers: alloc::vec::Vec) -> vortex_error::VortexResult -pub fn vortex_array::serde::ArrayParts::try_from(value: vortex_array::buffer::BufferHandle) -> core::result::Result +pub fn vortex_array::serde::SerializedArray::metadata(&self) -> &[u8] -impl core::convert::TryFrom> for vortex_array::serde::ArrayParts +pub fn vortex_array::serde::SerializedArray::nbuffers(&self) -> usize -pub type vortex_array::serde::ArrayParts::Error = vortex_error::VortexError +pub fn vortex_array::serde::SerializedArray::nchildren(&self) -> usize -pub fn vortex_array::serde::ArrayParts::try_from(value: vortex_buffer::ByteBuffer) -> core::result::Result +impl core::clone::Clone for vortex_array::serde::SerializedArray -impl core::fmt::Debug for vortex_array::serde::ArrayParts +pub fn vortex_array::serde::SerializedArray::clone(&self) -> vortex_array::serde::SerializedArray -pub fn vortex_array::serde::ArrayParts::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::convert::TryFrom for vortex_array::serde::SerializedArray -pub struct vortex_array::serde::SerializeOptions +pub type vortex_array::serde::SerializedArray::Error = vortex_error::VortexError -pub vortex_array::serde::SerializeOptions::include_padding: bool +pub fn vortex_array::serde::SerializedArray::try_from(value: vortex_array::buffer::BufferHandle) -> core::result::Result -pub vortex_array::serde::SerializeOptions::offset: usize +impl core::convert::TryFrom> for vortex_array::serde::SerializedArray -impl core::default::Default for vortex_array::serde::SerializeOptions +pub type vortex_array::serde::SerializedArray::Error = vortex_error::VortexError -pub fn vortex_array::serde::SerializeOptions::default() -> vortex_array::serde::SerializeOptions +pub fn vortex_array::serde::SerializedArray::try_from(value: vortex_buffer::ByteBuffer) -> core::result::Result -impl core::fmt::Debug for vortex_array::serde::SerializeOptions +impl core::fmt::Debug for vortex_array::serde::SerializedArray -pub fn vortex_array::serde::SerializeOptions::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_array::serde::SerializedArray::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub trait vortex_array::serde::ArrayChildren @@ -19408,9 +18726,7 @@ pub fn vortex_array::ValidityVTableFromValiditySliceHelper::validity(array: vort pub trait vortex_array::vtable::ArrayVTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug -pub type vortex_array::vtable::ArrayVTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug + vortex_array::IntoArray - -pub type vortex_array::vtable::ArrayVTable::Metadata: core::fmt::Debug +pub type vortex_array::vtable::ArrayVTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug pub type vortex_array::vtable::ArrayVTable::OperationsVTable: vortex_array::OperationsVTable @@ -19426,15 +18742,11 @@ pub fn vortex_array::vtable::ArrayVTable::buffer(array: vortex_array::ArrayView< pub fn vortex_array::vtable::ArrayVTable::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::vtable::ArrayVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::vtable::ArrayVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::vtable::ArrayVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::vtable::ArrayVTable::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::vtable::ArrayVTable::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::vtable::ArrayVTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::vtable::ArrayVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19442,10 +18754,6 @@ pub fn vortex_array::vtable::ArrayVTable::execute_parent(array: vortex_array::Ar pub fn vortex_array::vtable::ArrayVTable::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::vtable::ArrayVTable::len(array: &Self::ArrayData) -> usize - -pub fn vortex_array::vtable::ArrayVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::vtable::ArrayVTable::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::vtable::ArrayVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19454,15 +18762,13 @@ pub fn vortex_array::vtable::ArrayVTable::reduce(array: vortex_array::ArrayView< pub fn vortex_array::vtable::ArrayVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::vtable::ArrayVTable::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::vtable::ArrayVTable::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::vtable::ArrayVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::vtable::ArrayVTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] -pub fn vortex_array::vtable::ArrayVTable::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::vtable::ArrayVTable::vtable(array: &Self::ArrayData) -> &Self +pub fn vortex_array::vtable::ArrayVTable::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::vtable::ArrayVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19470,8 +18776,6 @@ impl vortex_array::VTable for vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ArrayData = vortex_array::arrays::bool::BoolData -pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Bool::OperationsVTable = vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool @@ -19486,15 +18790,11 @@ pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Bool::dtype(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19502,10 +18802,6 @@ pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Bool::len(array: &vortex_array::arrays::bool::BoolData) -> usize - -pub fn vortex_array::arrays::Bool::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Bool::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19514,15 +18810,13 @@ pub fn vortex_array::arrays::Bool::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Bool::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19530,8 +18824,6 @@ impl vortex_array::VTable for vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ArrayData = vortex_array::arrays::chunked::ChunkedData -pub type vortex_array::arrays::Chunked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Chunked::OperationsVTable = vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::Chunked @@ -19546,15 +18838,11 @@ pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Chunked::dtype(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19562,10 +18850,6 @@ pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayV pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Chunked::len(array: &vortex_array::arrays::chunked::ChunkedData) -> usize - -pub fn vortex_array::arrays::Chunked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Chunked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19574,15 +18858,13 @@ pub fn vortex_array::arrays::Chunked::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Chunked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19590,8 +18872,6 @@ impl vortex_array::VTable for vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ArrayData = vortex_array::arrays::constant::ConstantData -pub type vortex_array::arrays::Constant::Metadata = vortex_array::scalar::Scalar - pub type vortex_array::arrays::Constant::OperationsVTable = vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays::Constant @@ -19606,15 +18886,11 @@ pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Constant::dtype(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19622,10 +18898,6 @@ pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::Array pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Constant::len(array: &vortex_array::arrays::constant::ConstantData) -> usize - -pub fn vortex_array::arrays::Constant::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Constant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19634,15 +18906,13 @@ pub fn vortex_array::arrays::Constant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Constant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19650,8 +18920,6 @@ impl vortex_array::VTable for vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ArrayData = vortex_array::arrays::decimal::DecimalData -pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Decimal::OperationsVTable = vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::Decimal @@ -19666,15 +18934,11 @@ pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Decimal::dtype(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19682,10 +18946,6 @@ pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayV pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Decimal::len(array: &vortex_array::arrays::decimal::DecimalData) -> usize - -pub fn vortex_array::arrays::Decimal::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Decimal::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19694,15 +18954,13 @@ pub fn vortex_array::arrays::Decimal::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19710,8 +18968,6 @@ impl vortex_array::VTable for vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData -pub type vortex_array::arrays::Extension::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -19726,15 +18982,11 @@ pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Extension::dtype(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19742,10 +18994,6 @@ pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::Arra pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Extension::len(array: &vortex_array::arrays::extension::ExtensionData) -> usize - -pub fn vortex_array::arrays::Extension::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Extension::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19754,15 +19002,13 @@ pub fn vortex_array::arrays::Extension::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Extension::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Extension::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19770,8 +19016,6 @@ impl vortex_array::VTable for vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ArrayData = vortex_array::arrays::filter::FilterData -pub type vortex_array::arrays::Filter::Metadata = vortex_array::arrays::filter::vtable::FilterMetadata - pub type vortex_array::arrays::Filter::OperationsVTable = vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Filter @@ -19786,15 +19030,11 @@ pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::filter::vtable::FilterMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Filter::dtype(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19802,10 +19042,6 @@ pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Filter::len(array: &vortex_array::arrays::filter::FilterData) -> usize - -pub fn vortex_array::arrays::Filter::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Filter::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19814,15 +19050,13 @@ pub fn vortex_array::arrays::Filter::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Filter::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Filter::vtable(_array: &vortex_array::arrays::filter::FilterData) -> &Self +pub fn vortex_array::arrays::Filter::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19830,8 +19064,6 @@ impl vortex_array::VTable for vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ArrayData = vortex_array::arrays::fixed_size_list::FixedSizeListData -pub type vortex_array::arrays::FixedSizeList::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::FixedSizeList::OperationsVTable = vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arrays::FixedSizeList @@ -19846,15 +19078,11 @@ pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::FixedSizeList::dtype(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19862,10 +19090,6 @@ pub fn vortex_array::arrays::FixedSizeList::execute_parent(array: vortex_array:: pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::FixedSizeList::len(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> usize - -pub fn vortex_array::arrays::FixedSizeList::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::FixedSizeList::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19874,15 +19098,13 @@ pub fn vortex_array::arrays::FixedSizeList::reduce(array: vortex_array::ArrayVie pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::FixedSizeList::vtable(_array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &Self +pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19890,8 +19112,6 @@ impl vortex_array::VTable for vortex_array::arrays::List pub type vortex_array::arrays::List::ArrayData = vortex_array::arrays::list::ListData -pub type vortex_array::arrays::List::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::List::OperationsVTable = vortex_array::arrays::List pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List @@ -19906,15 +19126,11 @@ pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::List::dtype(array: &vortex_array::arrays::list::ListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19922,10 +19138,6 @@ pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::List::len(array: &vortex_array::arrays::list::ListData) -> usize - -pub fn vortex_array::arrays::List::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::List::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19934,15 +19146,13 @@ pub fn vortex_array::arrays::List::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::List::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::list::ListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::List::vtable(_array: &vortex_array::arrays::list::ListData) -> &Self +pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19950,8 +19160,6 @@ impl vortex_array::VTable for vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ArrayData = vortex_array::arrays::listview::ListViewData -pub type vortex_array::arrays::ListView::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::ListView::OperationsVTable = vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays::ListView @@ -19966,15 +19174,11 @@ pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::ListView::dtype(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -19982,10 +19186,6 @@ pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::Array pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::ListView::len(array: &vortex_array::arrays::listview::ListViewData) -> usize - -pub fn vortex_array::arrays::ListView::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::ListView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19994,15 +19194,13 @@ pub fn vortex_array::arrays::ListView::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::ListView::vtable(_array: &vortex_array::arrays::listview::ListViewData) -> &Self +pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20010,8 +19208,6 @@ impl vortex_array::VTable for vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ArrayData = vortex_array::arrays::masked::MaskedData -pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked @@ -20026,15 +19222,11 @@ pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Masked::dtype(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20042,10 +19234,6 @@ pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Masked::len(array: &vortex_array::arrays::masked::MaskedData) -> usize - -pub fn vortex_array::arrays::Masked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Masked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20054,15 +19242,13 @@ pub fn vortex_array::arrays::Masked::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Masked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Masked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20070,8 +19256,6 @@ impl vortex_array::VTable for vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ArrayData = vortex_array::arrays::primitive::PrimitiveData -pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive @@ -20086,15 +19270,11 @@ pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Primitive::dtype(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20102,10 +19282,6 @@ pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::Arra pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Primitive::len(array: &vortex_array::arrays::primitive::PrimitiveData) -> usize - -pub fn vortex_array::arrays::Primitive::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Primitive::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20114,15 +19290,13 @@ pub fn vortex_array::arrays::Primitive::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20130,8 +19304,6 @@ impl vortex_array::VTable for vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ArrayData = vortex_array::arrays::shared::SharedData -pub type vortex_array::arrays::Shared::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Shared::OperationsVTable = vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Shared @@ -20146,15 +19318,11 @@ pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Shared::dtype(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20162,10 +19330,6 @@ pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Shared::len(array: &vortex_array::arrays::shared::SharedData) -> usize - -pub fn vortex_array::arrays::Shared::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Shared::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20174,15 +19338,13 @@ pub fn vortex_array::arrays::Shared::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Shared::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Shared::vtable(_array: &vortex_array::arrays::shared::SharedData) -> &Self +pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20190,8 +19352,6 @@ impl vortex_array::VTable for vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ArrayData = vortex_array::arrays::struct_::StructData -pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct @@ -20206,15 +19366,11 @@ pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Struct::dtype(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20222,10 +19378,6 @@ pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Struct::len(array: &vortex_array::arrays::struct_::StructData) -> usize - -pub fn vortex_array::arrays::Struct::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Struct::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20234,15 +19386,13 @@ pub fn vortex_array::arrays::Struct::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Struct::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Struct::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20250,8 +19400,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ArrayData = vortex_array::arrays::varbin::VarBinData -pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::VarBin::OperationsVTable = vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::VarBin @@ -20266,15 +19414,11 @@ pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBin::dtype(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20282,10 +19426,6 @@ pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::VarBin::len(array: &vortex_array::arrays::varbin::VarBinData) -> usize - -pub fn vortex_array::arrays::VarBin::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBin::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20294,15 +19434,13 @@ pub fn vortex_array::arrays::VarBin::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBin::vtable(_array: &vortex_array::arrays::varbin::VarBinData) -> &Self +pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20310,8 +19448,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ArrayData = vortex_array::arrays::varbinview::VarBinViewData -pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView @@ -20326,15 +19462,11 @@ pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBinView::dtype(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20342,10 +19474,6 @@ pub fn vortex_array::arrays::VarBinView::execute_parent(array: vortex_array::Arr pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::VarBinView::len(array: &vortex_array::arrays::varbinview::VarBinViewData) -> usize - -pub fn vortex_array::arrays::VarBinView::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBinView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20354,15 +19482,13 @@ pub fn vortex_array::arrays::VarBinView::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBinView::vtable(_array: &vortex_array::arrays::varbinview::VarBinViewData) -> &Self +pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20370,8 +19496,6 @@ impl vortex_array::VTable for vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ArrayData = vortex_array::arrays::variant::VariantData -pub type vortex_array::arrays::Variant::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Variant::OperationsVTable = vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::Variant @@ -20386,15 +19510,11 @@ pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Variant::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20402,10 +19522,6 @@ pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayV pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Variant::len(array: &Self::ArrayData) -> usize - -pub fn vortex_array::arrays::Variant::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Variant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20414,15 +19530,13 @@ pub fn vortex_array::arrays::Variant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Variant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Variant::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20430,8 +19544,6 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData -pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -20446,15 +19558,11 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20462,10 +19570,6 @@ pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::Arr pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize - -pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::dict::Dict::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20474,15 +19578,13 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20490,8 +19592,6 @@ impl vortex_array::VTable for vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ArrayData = vortex_array::arrays::null::NullData -pub type vortex_array::arrays::null::Null::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::null::Null::OperationsVTable = vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays::null::Null @@ -20506,15 +19606,11 @@ pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::null::Null::dtype(_array: &vortex_array::arrays::null::NullData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20522,10 +19618,6 @@ pub fn vortex_array::arrays::null::Null::execute_parent(array: vortex_array::Arr pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::null::Null::len(array: &vortex_array::arrays::null::NullData) -> usize - -pub fn vortex_array::arrays::null::Null::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::null::Null::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20534,15 +19626,13 @@ pub fn vortex_array::arrays::null::Null::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::null::Null::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::null::Null::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20550,8 +19640,6 @@ impl vortex_array::VTable for vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray -pub type vortex_array::arrays::patched::Patched::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -20566,15 +19654,11 @@ pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::patched::Patched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20582,10 +19666,6 @@ pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_arra pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::patched::Patched::len(array: &Self::ArrayData) -> usize - -pub fn vortex_array::arrays::patched::Patched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::patched::Patched::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20594,15 +19674,13 @@ pub fn vortex_array::arrays::patched::Patched::reduce(array: vortex_array::Array pub fn vortex_array::arrays::patched::Patched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::patched::Patched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::patched::Patched::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20610,8 +19688,6 @@ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::Metadata = vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata - pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -20626,15 +19702,11 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::dtype(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20642,10 +19714,6 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vo pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::len(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> usize - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20654,15 +19722,13 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_arr pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::vtable(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &Self +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20670,8 +19736,6 @@ impl vortex_array::VTable for vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ArrayData = vortex_array::arrays::slice::SliceData -pub type vortex_array::arrays::slice::Slice::Metadata = vortex_array::arrays::slice::SliceMetadata - pub type vortex_array::arrays::slice::Slice::OperationsVTable = vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arrays::slice::Slice @@ -20686,15 +19750,11 @@ pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::slice::SliceMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20702,10 +19762,6 @@ pub fn vortex_array::arrays::slice::Slice::execute_parent(array: vortex_array::A pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::slice::Slice::len(array: &vortex_array::arrays::slice::SliceData) -> usize - -pub fn vortex_array::arrays::slice::Slice::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::slice::Slice::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20714,15 +19770,13 @@ pub fn vortex_array::arrays::slice::Slice::reduce(array: vortex_array::ArrayView pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::slice::Slice::vtable(_array: &vortex_array::arrays::slice::SliceData) -> &Self +pub fn vortex_array::arrays::slice::Slice::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20852,9 +19906,7 @@ pub fn vortex_array::NotSupported::scalar_at(array: vortex_array::ArrayView<'_, pub trait vortex_array::vtable::VTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug -pub type vortex_array::vtable::VTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug + vortex_array::IntoArray - -pub type vortex_array::vtable::VTable::Metadata: core::fmt::Debug +pub type vortex_array::vtable::VTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug pub type vortex_array::vtable::VTable::OperationsVTable: vortex_array::OperationsVTable @@ -20870,15 +19922,11 @@ pub fn vortex_array::vtable::VTable::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::vtable::VTable::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::vtable::VTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::vtable::VTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::vtable::VTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::vtable::VTable::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::vtable::VTable::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::vtable::VTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::vtable::VTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20886,10 +19934,6 @@ pub fn vortex_array::vtable::VTable::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::vtable::VTable::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::vtable::VTable::len(array: &Self::ArrayData) -> usize - -pub fn vortex_array::vtable::VTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::vtable::VTable::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::vtable::VTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20898,15 +19942,13 @@ pub fn vortex_array::vtable::VTable::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::vtable::VTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::vtable::VTable::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::vtable::VTable::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::vtable::VTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::vtable::VTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] -pub fn vortex_array::vtable::VTable::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::vtable::VTable::vtable(array: &Self::ArrayData) -> &Self +pub fn vortex_array::vtable::VTable::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::vtable::VTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20914,8 +19956,6 @@ impl vortex_array::VTable for vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ArrayData = vortex_array::arrays::bool::BoolData -pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Bool::OperationsVTable = vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool @@ -20930,15 +19970,11 @@ pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Bool::dtype(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20946,10 +19982,6 @@ pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Bool::len(array: &vortex_array::arrays::bool::BoolData) -> usize - -pub fn vortex_array::arrays::Bool::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Bool::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20958,15 +19990,13 @@ pub fn vortex_array::arrays::Bool::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Bool::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20974,8 +20004,6 @@ impl vortex_array::VTable for vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ArrayData = vortex_array::arrays::chunked::ChunkedData -pub type vortex_array::arrays::Chunked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Chunked::OperationsVTable = vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::Chunked @@ -20990,15 +20018,11 @@ pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Chunked::dtype(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21006,10 +20030,6 @@ pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayV pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Chunked::len(array: &vortex_array::arrays::chunked::ChunkedData) -> usize - -pub fn vortex_array::arrays::Chunked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Chunked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21018,15 +20038,13 @@ pub fn vortex_array::arrays::Chunked::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Chunked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21034,8 +20052,6 @@ impl vortex_array::VTable for vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ArrayData = vortex_array::arrays::constant::ConstantData -pub type vortex_array::arrays::Constant::Metadata = vortex_array::scalar::Scalar - pub type vortex_array::arrays::Constant::OperationsVTable = vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays::Constant @@ -21050,15 +20066,11 @@ pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Constant::dtype(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21066,10 +20078,6 @@ pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::Array pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Constant::len(array: &vortex_array::arrays::constant::ConstantData) -> usize - -pub fn vortex_array::arrays::Constant::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Constant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21078,15 +20086,13 @@ pub fn vortex_array::arrays::Constant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Constant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21094,8 +20100,6 @@ impl vortex_array::VTable for vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ArrayData = vortex_array::arrays::decimal::DecimalData -pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Decimal::OperationsVTable = vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::Decimal @@ -21110,15 +20114,11 @@ pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Decimal::dtype(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21126,10 +20126,6 @@ pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayV pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Decimal::len(array: &vortex_array::arrays::decimal::DecimalData) -> usize - -pub fn vortex_array::arrays::Decimal::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Decimal::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21138,15 +20134,13 @@ pub fn vortex_array::arrays::Decimal::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21154,8 +20148,6 @@ impl vortex_array::VTable for vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData -pub type vortex_array::arrays::Extension::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -21170,15 +20162,11 @@ pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Extension::dtype(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21186,10 +20174,6 @@ pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::Arra pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Extension::len(array: &vortex_array::arrays::extension::ExtensionData) -> usize - -pub fn vortex_array::arrays::Extension::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Extension::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21198,15 +20182,13 @@ pub fn vortex_array::arrays::Extension::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Extension::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Extension::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21214,8 +20196,6 @@ impl vortex_array::VTable for vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ArrayData = vortex_array::arrays::filter::FilterData -pub type vortex_array::arrays::Filter::Metadata = vortex_array::arrays::filter::vtable::FilterMetadata - pub type vortex_array::arrays::Filter::OperationsVTable = vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Filter @@ -21230,15 +20210,11 @@ pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::filter::vtable::FilterMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Filter::dtype(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21246,10 +20222,6 @@ pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Filter::len(array: &vortex_array::arrays::filter::FilterData) -> usize - -pub fn vortex_array::arrays::Filter::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Filter::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21258,15 +20230,13 @@ pub fn vortex_array::arrays::Filter::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Filter::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Filter::vtable(_array: &vortex_array::arrays::filter::FilterData) -> &Self +pub fn vortex_array::arrays::Filter::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21274,8 +20244,6 @@ impl vortex_array::VTable for vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ArrayData = vortex_array::arrays::fixed_size_list::FixedSizeListData -pub type vortex_array::arrays::FixedSizeList::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::FixedSizeList::OperationsVTable = vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arrays::FixedSizeList @@ -21290,15 +20258,11 @@ pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::FixedSizeList::dtype(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21306,10 +20270,6 @@ pub fn vortex_array::arrays::FixedSizeList::execute_parent(array: vortex_array:: pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::FixedSizeList::len(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> usize - -pub fn vortex_array::arrays::FixedSizeList::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::FixedSizeList::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21318,15 +20278,13 @@ pub fn vortex_array::arrays::FixedSizeList::reduce(array: vortex_array::ArrayVie pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::FixedSizeList::vtable(_array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &Self +pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21334,8 +20292,6 @@ impl vortex_array::VTable for vortex_array::arrays::List pub type vortex_array::arrays::List::ArrayData = vortex_array::arrays::list::ListData -pub type vortex_array::arrays::List::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::List::OperationsVTable = vortex_array::arrays::List pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List @@ -21350,15 +20306,11 @@ pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::List::dtype(array: &vortex_array::arrays::list::ListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21366,10 +20318,6 @@ pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::List::len(array: &vortex_array::arrays::list::ListData) -> usize - -pub fn vortex_array::arrays::List::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::List::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21378,15 +20326,13 @@ pub fn vortex_array::arrays::List::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::List::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::list::ListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::List::vtable(_array: &vortex_array::arrays::list::ListData) -> &Self +pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21394,8 +20340,6 @@ impl vortex_array::VTable for vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ArrayData = vortex_array::arrays::listview::ListViewData -pub type vortex_array::arrays::ListView::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::ListView::OperationsVTable = vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays::ListView @@ -21410,15 +20354,11 @@ pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::ListView::dtype(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21426,10 +20366,6 @@ pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::Array pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::ListView::len(array: &vortex_array::arrays::listview::ListViewData) -> usize - -pub fn vortex_array::arrays::ListView::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::ListView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21438,15 +20374,13 @@ pub fn vortex_array::arrays::ListView::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::ListView::vtable(_array: &vortex_array::arrays::listview::ListViewData) -> &Self +pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21454,8 +20388,6 @@ impl vortex_array::VTable for vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ArrayData = vortex_array::arrays::masked::MaskedData -pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked @@ -21470,15 +20402,11 @@ pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Masked::dtype(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21486,10 +20414,6 @@ pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Masked::len(array: &vortex_array::arrays::masked::MaskedData) -> usize - -pub fn vortex_array::arrays::Masked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Masked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21498,15 +20422,13 @@ pub fn vortex_array::arrays::Masked::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Masked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Masked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21514,8 +20436,6 @@ impl vortex_array::VTable for vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ArrayData = vortex_array::arrays::primitive::PrimitiveData -pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive @@ -21530,15 +20450,11 @@ pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Primitive::dtype(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21546,10 +20462,6 @@ pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::Arra pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Primitive::len(array: &vortex_array::arrays::primitive::PrimitiveData) -> usize - -pub fn vortex_array::arrays::Primitive::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Primitive::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21558,15 +20470,13 @@ pub fn vortex_array::arrays::Primitive::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21574,8 +20484,6 @@ impl vortex_array::VTable for vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ArrayData = vortex_array::arrays::shared::SharedData -pub type vortex_array::arrays::Shared::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Shared::OperationsVTable = vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Shared @@ -21590,15 +20498,11 @@ pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Shared::dtype(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21606,10 +20510,6 @@ pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Shared::len(array: &vortex_array::arrays::shared::SharedData) -> usize - -pub fn vortex_array::arrays::Shared::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Shared::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21618,15 +20518,13 @@ pub fn vortex_array::arrays::Shared::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Shared::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Shared::vtable(_array: &vortex_array::arrays::shared::SharedData) -> &Self +pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21634,8 +20532,6 @@ impl vortex_array::VTable for vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ArrayData = vortex_array::arrays::struct_::StructData -pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct @@ -21650,15 +20546,11 @@ pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Struct::dtype(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21666,10 +20558,6 @@ pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Struct::len(array: &vortex_array::arrays::struct_::StructData) -> usize - -pub fn vortex_array::arrays::Struct::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Struct::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21678,15 +20566,13 @@ pub fn vortex_array::arrays::Struct::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Struct::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Struct::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21694,8 +20580,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ArrayData = vortex_array::arrays::varbin::VarBinData -pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::VarBin::OperationsVTable = vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::VarBin @@ -21710,15 +20594,11 @@ pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBin::dtype(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21726,10 +20606,6 @@ pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::VarBin::len(array: &vortex_array::arrays::varbin::VarBinData) -> usize - -pub fn vortex_array::arrays::VarBin::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBin::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21738,15 +20614,13 @@ pub fn vortex_array::arrays::VarBin::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBin::vtable(_array: &vortex_array::arrays::varbin::VarBinData) -> &Self +pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21754,8 +20628,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ArrayData = vortex_array::arrays::varbinview::VarBinViewData -pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView @@ -21770,15 +20642,11 @@ pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBinView::dtype(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21786,10 +20654,6 @@ pub fn vortex_array::arrays::VarBinView::execute_parent(array: vortex_array::Arr pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::VarBinView::len(array: &vortex_array::arrays::varbinview::VarBinViewData) -> usize - -pub fn vortex_array::arrays::VarBinView::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBinView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21798,15 +20662,13 @@ pub fn vortex_array::arrays::VarBinView::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBinView::vtable(_array: &vortex_array::arrays::varbinview::VarBinViewData) -> &Self +pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21814,8 +20676,6 @@ impl vortex_array::VTable for vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ArrayData = vortex_array::arrays::variant::VariantData -pub type vortex_array::arrays::Variant::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Variant::OperationsVTable = vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::Variant @@ -21830,15 +20690,11 @@ pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Variant::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21846,10 +20702,6 @@ pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayV pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Variant::len(array: &Self::ArrayData) -> usize - -pub fn vortex_array::arrays::Variant::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Variant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21858,15 +20710,13 @@ pub fn vortex_array::arrays::Variant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Variant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Variant::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21874,8 +20724,6 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData -pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -21890,15 +20738,11 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21906,10 +20750,6 @@ pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::Arr pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize - -pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::dict::Dict::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21918,15 +20758,13 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21934,8 +20772,6 @@ impl vortex_array::VTable for vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ArrayData = vortex_array::arrays::null::NullData -pub type vortex_array::arrays::null::Null::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::null::Null::OperationsVTable = vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays::null::Null @@ -21950,15 +20786,11 @@ pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::null::Null::dtype(_array: &vortex_array::arrays::null::NullData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21966,10 +20798,6 @@ pub fn vortex_array::arrays::null::Null::execute_parent(array: vortex_array::Arr pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::null::Null::len(array: &vortex_array::arrays::null::NullData) -> usize - -pub fn vortex_array::arrays::null::Null::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::null::Null::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21978,15 +20806,13 @@ pub fn vortex_array::arrays::null::Null::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::null::Null::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::null::Null::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21994,8 +20820,6 @@ impl vortex_array::VTable for vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray -pub type vortex_array::arrays::patched::Patched::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -22010,15 +20834,11 @@ pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::patched::Patched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -22026,10 +20846,6 @@ pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_arra pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::patched::Patched::len(array: &Self::ArrayData) -> usize - -pub fn vortex_array::arrays::patched::Patched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::patched::Patched::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22038,15 +20854,13 @@ pub fn vortex_array::arrays::patched::Patched::reduce(array: vortex_array::Array pub fn vortex_array::arrays::patched::Patched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::patched::Patched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::patched::Patched::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -22054,8 +20868,6 @@ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::Metadata = vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata - pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -22070,15 +20882,11 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::dtype(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -22086,10 +20894,6 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vo pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::len(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> usize - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22098,15 +20902,13 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_arr pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::vtable(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &Self +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -22114,8 +20916,6 @@ impl vortex_array::VTable for vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ArrayData = vortex_array::arrays::slice::SliceData -pub type vortex_array::arrays::slice::Slice::Metadata = vortex_array::arrays::slice::SliceMetadata - pub type vortex_array::arrays::slice::Slice::OperationsVTable = vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arrays::slice::Slice @@ -22130,15 +20930,11 @@ pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::slice::SliceMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -22146,10 +20942,6 @@ pub fn vortex_array::arrays::slice::Slice::execute_parent(array: vortex_array::A pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::slice::Slice::len(array: &vortex_array::arrays::slice::SliceData) -> usize - -pub fn vortex_array::arrays::slice::Slice::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::slice::Slice::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22158,15 +20950,13 @@ pub fn vortex_array::arrays::slice::Slice::reduce(array: vortex_array::ArrayView pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::slice::Slice::vtable(_array: &vortex_array::arrays::slice::SliceData) -> &Self +pub fn vortex_array::arrays::slice::Slice::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -22562,7 +21352,7 @@ pub fn vortex_array::Array::from_indices::into_bit_buffer(self) -> vortex_buffer::bit::buf::BitBuffer -pub fn vortex_array::Array::into_parts(self) -> vortex_array::arrays::bool::BoolArrayParts +pub fn vortex_array::Array::into_encoding_parts(self) -> vortex_array::arrays::bool::BoolArrayParts pub fn vortex_array::Array::new(bits: vortex_buffer::bit::buf::BitBuffer, validity: vortex_array::validity::Validity) -> Self @@ -22584,6 +21374,8 @@ impl vortex_array::Array pub unsafe fn vortex_array::Array::new_unchecked(chunks: alloc::vec::Vec, dtype: vortex_array::dtype::DType) -> Self +pub fn vortex_array::Array::rechunk(&self, target_bytesize: u64, target_rowsize: usize) -> vortex_error::VortexResult + pub fn vortex_array::Array::try_new(chunks: alloc::vec::Vec, dtype: vortex_array::dtype::DType) -> vortex_error::VortexResult impl vortex_array::Array @@ -22704,7 +21496,7 @@ impl vortex_array::Array pub fn vortex_array::Array::from_fields>(items: &[(N, vortex_array::ArrayRef)]) -> vortex_error::VortexResult -pub fn vortex_array::Array::into_parts(self) -> vortex_array::arrays::struct_::StructArrayParts +pub fn vortex_array::Array::into_encoding_parts(self) -> vortex_array::arrays::struct_::StructArrayParts pub fn vortex_array::Array::new(names: vortex_array::dtype::FieldNames, fields: impl core::convert::Into>, length: usize, validity: vortex_array::validity::Validity) -> Self @@ -22860,6 +21652,8 @@ pub fn vortex_array::Array::encoding_id(&self) -> vortex_array::ArrayId pub fn vortex_array::Array::into_data(self) -> ::ArrayData +pub fn vortex_array::Array::into_parts(self) -> vortex_array::ArrayParts + pub fn vortex_array::Array::is_empty(&self) -> bool pub fn vortex_array::Array::len(&self) -> usize @@ -22868,7 +21662,7 @@ pub fn vortex_array::Array::statistics(&self) -> vortex_array::stats::StatsSe pub fn vortex_array::Array::try_from_array_ref(array: vortex_array::ArrayRef) -> core::result::Result -pub fn vortex_array::Array::try_from_data(data: ::ArrayData) -> vortex_error::VortexResult +pub fn vortex_array::Array::try_from_parts(new: vortex_array::ArrayParts) -> vortex_error::VortexResult impl core::convert::From> for vortex_array::Array @@ -22970,6 +21764,24 @@ impl vortex_array::IntoArray for vortex_array::Array pub fn vortex_array::Array::into_array(self) -> vortex_array::ArrayRef +pub struct vortex_array::ArrayParts + +pub vortex_array::ArrayParts::data: ::ArrayData + +pub vortex_array::ArrayParts::dtype: vortex_array::dtype::DType + +pub vortex_array::ArrayParts::len: usize + +pub vortex_array::ArrayParts::stats: vortex_array::stats::ArrayStats + +pub vortex_array::ArrayParts::vtable: V + +impl vortex_array::ArrayParts + +pub fn vortex_array::ArrayParts::new(vtable: V, dtype: vortex_array::dtype::DType, len: usize, data: ::ArrayData) -> Self + +pub fn vortex_array::ArrayParts::with_stats(self, stats: vortex_array::stats::ArrayStats) -> Self + pub struct vortex_array::ArrayRef(_) impl vortex_array::ArrayRef @@ -23028,8 +21840,6 @@ pub fn vortex_array::ArrayRef::len(&self) -> usize pub fn vortex_array::ArrayRef::metadata(&self) -> vortex_error::VortexResult>> -pub fn vortex_array::ArrayRef::metadata_fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::ArrayRef::named_buffers(&self) -> alloc::vec::Vec<(alloc::string::String, vortex_array::buffer::BufferHandle)> pub fn vortex_array::ArrayRef::named_children(&self) -> alloc::vec::Vec<(alloc::string::String, vortex_array::ArrayRef)> @@ -23146,93 +21956,13 @@ impl core::convert::From for vortex_array::ArrayRef pub fn vortex_array::ArrayRef::from(value: vortex_array::Canonical) -> Self -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::bool::BoolData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::chunked::ChunkedData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::constant::ConstantData) -> vortex_array::ArrayRef - impl core::convert::From for vortex_array::ArrayRef pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::datetime::TemporalData) -> Self -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::decimal::DecimalData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::dict::DictData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::extension::ExtensionData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::filter::FilterData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::fixed_size_list::FixedSizeListData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::list::ListData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::listview::ListViewData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::masked::MaskedData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::null::NullData) -> vortex_array::ArrayRef - impl core::convert::From for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::primitive::PrimitiveData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::scalar_fn::ScalarFnData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::shared::SharedData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::slice::SliceData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::struct_::StructData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::varbin::VarBinData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::varbinview::VarBinViewData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::variant::VariantData) -> vortex_array::ArrayRef +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> Self impl core::convert::TryFrom<&vortex_array::ArrayRef> for arrow_array::record_batch::RecordBatch @@ -23814,9 +22544,7 @@ pub fn core::option::Option::array_hash(&self, state: pub trait vortex_array::ArrayVTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug -pub type vortex_array::ArrayVTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug + vortex_array::IntoArray - -pub type vortex_array::ArrayVTable::Metadata: core::fmt::Debug +pub type vortex_array::ArrayVTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug pub type vortex_array::ArrayVTable::OperationsVTable: vortex_array::OperationsVTable @@ -23832,15 +22560,11 @@ pub fn vortex_array::ArrayVTable::buffer(array: vortex_array::ArrayView<'_, Self pub fn vortex_array::ArrayVTable::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::ArrayVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::ArrayVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::ArrayVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::ArrayVTable::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::ArrayVTable::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::ArrayVTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::ArrayVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23848,10 +22572,6 @@ pub fn vortex_array::ArrayVTable::execute_parent(array: vortex_array::ArrayView< pub fn vortex_array::ArrayVTable::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::ArrayVTable::len(array: &Self::ArrayData) -> usize - -pub fn vortex_array::ArrayVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::ArrayVTable::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::ArrayVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23860,15 +22580,13 @@ pub fn vortex_array::ArrayVTable::reduce(array: vortex_array::ArrayView<'_, Self pub fn vortex_array::ArrayVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::ArrayVTable::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::ArrayVTable::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::ArrayVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::ArrayVTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] -pub fn vortex_array::ArrayVTable::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::ArrayVTable::vtable(array: &Self::ArrayData) -> &Self +pub fn vortex_array::ArrayVTable::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::ArrayVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23876,8 +22594,6 @@ impl vortex_array::VTable for vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ArrayData = vortex_array::arrays::bool::BoolData -pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Bool::OperationsVTable = vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool @@ -23892,15 +22608,11 @@ pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Bool::dtype(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23908,10 +22620,6 @@ pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Bool::len(array: &vortex_array::arrays::bool::BoolData) -> usize - -pub fn vortex_array::arrays::Bool::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Bool::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23920,15 +22628,13 @@ pub fn vortex_array::arrays::Bool::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Bool::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23936,8 +22642,6 @@ impl vortex_array::VTable for vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ArrayData = vortex_array::arrays::chunked::ChunkedData -pub type vortex_array::arrays::Chunked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Chunked::OperationsVTable = vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::Chunked @@ -23952,15 +22656,11 @@ pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Chunked::dtype(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -23968,10 +22668,6 @@ pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayV pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Chunked::len(array: &vortex_array::arrays::chunked::ChunkedData) -> usize - -pub fn vortex_array::arrays::Chunked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Chunked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23980,15 +22676,13 @@ pub fn vortex_array::arrays::Chunked::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Chunked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23996,8 +22690,6 @@ impl vortex_array::VTable for vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ArrayData = vortex_array::arrays::constant::ConstantData -pub type vortex_array::arrays::Constant::Metadata = vortex_array::scalar::Scalar - pub type vortex_array::arrays::Constant::OperationsVTable = vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays::Constant @@ -24012,15 +22704,11 @@ pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Constant::dtype(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24028,10 +22716,6 @@ pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::Array pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Constant::len(array: &vortex_array::arrays::constant::ConstantData) -> usize - -pub fn vortex_array::arrays::Constant::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Constant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24040,15 +22724,13 @@ pub fn vortex_array::arrays::Constant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Constant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24056,8 +22738,6 @@ impl vortex_array::VTable for vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ArrayData = vortex_array::arrays::decimal::DecimalData -pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Decimal::OperationsVTable = vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::Decimal @@ -24072,15 +22752,11 @@ pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Decimal::dtype(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24088,10 +22764,6 @@ pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayV pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Decimal::len(array: &vortex_array::arrays::decimal::DecimalData) -> usize - -pub fn vortex_array::arrays::Decimal::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Decimal::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24100,15 +22772,13 @@ pub fn vortex_array::arrays::Decimal::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24116,8 +22786,6 @@ impl vortex_array::VTable for vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData -pub type vortex_array::arrays::Extension::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -24132,15 +22800,11 @@ pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Extension::dtype(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24148,10 +22812,6 @@ pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::Arra pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Extension::len(array: &vortex_array::arrays::extension::ExtensionData) -> usize - -pub fn vortex_array::arrays::Extension::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Extension::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24160,15 +22820,13 @@ pub fn vortex_array::arrays::Extension::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Extension::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Extension::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24176,8 +22834,6 @@ impl vortex_array::VTable for vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ArrayData = vortex_array::arrays::filter::FilterData -pub type vortex_array::arrays::Filter::Metadata = vortex_array::arrays::filter::vtable::FilterMetadata - pub type vortex_array::arrays::Filter::OperationsVTable = vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Filter @@ -24192,15 +22848,11 @@ pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::filter::vtable::FilterMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Filter::dtype(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24208,10 +22860,6 @@ pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Filter::len(array: &vortex_array::arrays::filter::FilterData) -> usize - -pub fn vortex_array::arrays::Filter::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Filter::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24220,15 +22868,13 @@ pub fn vortex_array::arrays::Filter::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Filter::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Filter::vtable(_array: &vortex_array::arrays::filter::FilterData) -> &Self +pub fn vortex_array::arrays::Filter::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24236,8 +22882,6 @@ impl vortex_array::VTable for vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ArrayData = vortex_array::arrays::fixed_size_list::FixedSizeListData -pub type vortex_array::arrays::FixedSizeList::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::FixedSizeList::OperationsVTable = vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arrays::FixedSizeList @@ -24252,15 +22896,11 @@ pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::FixedSizeList::dtype(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24268,10 +22908,6 @@ pub fn vortex_array::arrays::FixedSizeList::execute_parent(array: vortex_array:: pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::FixedSizeList::len(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> usize - -pub fn vortex_array::arrays::FixedSizeList::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::FixedSizeList::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24280,15 +22916,13 @@ pub fn vortex_array::arrays::FixedSizeList::reduce(array: vortex_array::ArrayVie pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::FixedSizeList::vtable(_array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &Self +pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24296,8 +22930,6 @@ impl vortex_array::VTable for vortex_array::arrays::List pub type vortex_array::arrays::List::ArrayData = vortex_array::arrays::list::ListData -pub type vortex_array::arrays::List::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::List::OperationsVTable = vortex_array::arrays::List pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List @@ -24312,15 +22944,11 @@ pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::List::dtype(array: &vortex_array::arrays::list::ListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24328,10 +22956,6 @@ pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::List::len(array: &vortex_array::arrays::list::ListData) -> usize - -pub fn vortex_array::arrays::List::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::List::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24340,15 +22964,13 @@ pub fn vortex_array::arrays::List::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::List::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::list::ListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::List::vtable(_array: &vortex_array::arrays::list::ListData) -> &Self +pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24356,8 +22978,6 @@ impl vortex_array::VTable for vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ArrayData = vortex_array::arrays::listview::ListViewData -pub type vortex_array::arrays::ListView::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::ListView::OperationsVTable = vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays::ListView @@ -24372,15 +22992,11 @@ pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::ListView::dtype(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24388,10 +23004,6 @@ pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::Array pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::ListView::len(array: &vortex_array::arrays::listview::ListViewData) -> usize - -pub fn vortex_array::arrays::ListView::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::ListView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24400,15 +23012,13 @@ pub fn vortex_array::arrays::ListView::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::ListView::vtable(_array: &vortex_array::arrays::listview::ListViewData) -> &Self +pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24416,8 +23026,6 @@ impl vortex_array::VTable for vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ArrayData = vortex_array::arrays::masked::MaskedData -pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked @@ -24432,15 +23040,11 @@ pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Masked::dtype(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24448,10 +23052,6 @@ pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Masked::len(array: &vortex_array::arrays::masked::MaskedData) -> usize - -pub fn vortex_array::arrays::Masked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Masked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24460,15 +23060,13 @@ pub fn vortex_array::arrays::Masked::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Masked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Masked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24476,8 +23074,6 @@ impl vortex_array::VTable for vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ArrayData = vortex_array::arrays::primitive::PrimitiveData -pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive @@ -24492,15 +23088,11 @@ pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Primitive::dtype(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24508,10 +23100,6 @@ pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::Arra pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Primitive::len(array: &vortex_array::arrays::primitive::PrimitiveData) -> usize - -pub fn vortex_array::arrays::Primitive::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Primitive::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24520,15 +23108,13 @@ pub fn vortex_array::arrays::Primitive::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24536,8 +23122,6 @@ impl vortex_array::VTable for vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ArrayData = vortex_array::arrays::shared::SharedData -pub type vortex_array::arrays::Shared::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Shared::OperationsVTable = vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Shared @@ -24552,15 +23136,11 @@ pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Shared::dtype(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24568,10 +23148,6 @@ pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Shared::len(array: &vortex_array::arrays::shared::SharedData) -> usize - -pub fn vortex_array::arrays::Shared::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Shared::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24580,15 +23156,13 @@ pub fn vortex_array::arrays::Shared::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Shared::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Shared::vtable(_array: &vortex_array::arrays::shared::SharedData) -> &Self +pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24596,8 +23170,6 @@ impl vortex_array::VTable for vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ArrayData = vortex_array::arrays::struct_::StructData -pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct @@ -24612,15 +23184,11 @@ pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Struct::dtype(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24628,10 +23196,6 @@ pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Struct::len(array: &vortex_array::arrays::struct_::StructData) -> usize - -pub fn vortex_array::arrays::Struct::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Struct::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24640,15 +23204,13 @@ pub fn vortex_array::arrays::Struct::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Struct::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Struct::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24656,8 +23218,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ArrayData = vortex_array::arrays::varbin::VarBinData -pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::VarBin::OperationsVTable = vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::VarBin @@ -24672,15 +23232,11 @@ pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBin::dtype(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24688,10 +23244,6 @@ pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::VarBin::len(array: &vortex_array::arrays::varbin::VarBinData) -> usize - -pub fn vortex_array::arrays::VarBin::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBin::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24700,15 +23252,13 @@ pub fn vortex_array::arrays::VarBin::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBin::vtable(_array: &vortex_array::arrays::varbin::VarBinData) -> &Self +pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24716,8 +23266,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ArrayData = vortex_array::arrays::varbinview::VarBinViewData -pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView @@ -24732,15 +23280,11 @@ pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBinView::dtype(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24748,10 +23292,6 @@ pub fn vortex_array::arrays::VarBinView::execute_parent(array: vortex_array::Arr pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::VarBinView::len(array: &vortex_array::arrays::varbinview::VarBinViewData) -> usize - -pub fn vortex_array::arrays::VarBinView::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBinView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24760,15 +23300,13 @@ pub fn vortex_array::arrays::VarBinView::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBinView::vtable(_array: &vortex_array::arrays::varbinview::VarBinViewData) -> &Self +pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24776,8 +23314,6 @@ impl vortex_array::VTable for vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ArrayData = vortex_array::arrays::variant::VariantData -pub type vortex_array::arrays::Variant::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Variant::OperationsVTable = vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::Variant @@ -24792,15 +23328,11 @@ pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Variant::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24808,10 +23340,6 @@ pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayV pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Variant::len(array: &Self::ArrayData) -> usize - -pub fn vortex_array::arrays::Variant::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Variant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24820,15 +23348,13 @@ pub fn vortex_array::arrays::Variant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Variant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Variant::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24836,8 +23362,6 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData -pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -24852,15 +23376,11 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24868,10 +23388,6 @@ pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::Arr pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize - -pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::dict::Dict::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24880,15 +23396,13 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24896,8 +23410,6 @@ impl vortex_array::VTable for vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ArrayData = vortex_array::arrays::null::NullData -pub type vortex_array::arrays::null::Null::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::null::Null::OperationsVTable = vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays::null::Null @@ -24912,15 +23424,11 @@ pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::null::Null::dtype(_array: &vortex_array::arrays::null::NullData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24928,10 +23436,6 @@ pub fn vortex_array::arrays::null::Null::execute_parent(array: vortex_array::Arr pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::null::Null::len(array: &vortex_array::arrays::null::NullData) -> usize - -pub fn vortex_array::arrays::null::Null::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::null::Null::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24940,15 +23444,13 @@ pub fn vortex_array::arrays::null::Null::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::null::Null::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::null::Null::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24956,8 +23458,6 @@ impl vortex_array::VTable for vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray -pub type vortex_array::arrays::patched::Patched::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -24972,15 +23472,11 @@ pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::patched::Patched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24988,10 +23484,6 @@ pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_arra pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::patched::Patched::len(array: &Self::ArrayData) -> usize - -pub fn vortex_array::arrays::patched::Patched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::patched::Patched::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25000,15 +23492,13 @@ pub fn vortex_array::arrays::patched::Patched::reduce(array: vortex_array::Array pub fn vortex_array::arrays::patched::Patched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::patched::Patched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::patched::Patched::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25016,8 +23506,6 @@ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::Metadata = vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata - pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -25032,15 +23520,11 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::dtype(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -25048,10 +23532,6 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vo pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::len(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> usize - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25060,15 +23540,13 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_arr pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::vtable(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &Self +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25076,8 +23554,6 @@ impl vortex_array::VTable for vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ArrayData = vortex_array::arrays::slice::SliceData -pub type vortex_array::arrays::slice::Slice::Metadata = vortex_array::arrays::slice::SliceMetadata - pub type vortex_array::arrays::slice::Slice::OperationsVTable = vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arrays::slice::Slice @@ -25092,15 +23568,11 @@ pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::slice::SliceMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -25108,10 +23580,6 @@ pub fn vortex_array::arrays::slice::Slice::execute_parent(array: vortex_array::A pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::slice::Slice::len(array: &vortex_array::arrays::slice::SliceData) -> usize - -pub fn vortex_array::arrays::slice::Slice::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::slice::Slice::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25120,15 +23588,13 @@ pub fn vortex_array::arrays::slice::Slice::reduce(array: vortex_array::ArrayView pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::slice::Slice::vtable(_array: &vortex_array::arrays::slice::SliceData) -> &Self +pub fn vortex_array::arrays::slice::Slice::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25304,94 +23770,14 @@ impl vortex_array::IntoArray for vortex_array::Columnar pub fn vortex_array::Columnar::into_array(self) -> vortex_array::ArrayRef -impl vortex_array::IntoArray for vortex_array::arrays::bool::BoolData - -pub fn vortex_array::arrays::bool::BoolData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::chunked::ChunkedData - -pub fn vortex_array::arrays::chunked::ChunkedData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::constant::ConstantData - -pub fn vortex_array::arrays::constant::ConstantData::into_array(self) -> vortex_array::ArrayRef - impl vortex_array::IntoArray for vortex_array::arrays::datetime::TemporalData pub fn vortex_array::arrays::datetime::TemporalData::into_array(self) -> vortex_array::ArrayRef -impl vortex_array::IntoArray for vortex_array::arrays::decimal::DecimalData - -pub fn vortex_array::arrays::decimal::DecimalData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::dict::DictData - -pub fn vortex_array::arrays::dict::DictData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::filter::FilterData - -pub fn vortex_array::arrays::filter::FilterData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::fixed_size_list::FixedSizeListData - -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::list::ListData - -pub fn vortex_array::arrays::list::ListData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::listview::ListViewData - -pub fn vortex_array::arrays::listview::ListViewData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::masked::MaskedData - -pub fn vortex_array::arrays::masked::MaskedData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::null::NullData - -pub fn vortex_array::arrays::null::NullData::into_array(self) -> vortex_array::ArrayRef - impl vortex_array::IntoArray for vortex_array::arrays::patched::PatchedArray pub fn vortex_array::arrays::patched::PatchedArray::into_array(self) -> vortex_array::ArrayRef -impl vortex_array::IntoArray for vortex_array::arrays::primitive::PrimitiveData - -pub fn vortex_array::arrays::primitive::PrimitiveData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::scalar_fn::ScalarFnData - -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::shared::SharedData - -pub fn vortex_array::arrays::shared::SharedData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::slice::SliceData - -pub fn vortex_array::arrays::slice::SliceData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::struct_::StructData - -pub fn vortex_array::arrays::struct_::StructData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::varbin::VarBinData - -pub fn vortex_array::arrays::varbin::VarBinData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::varbinview::VarBinViewData - -pub fn vortex_array::arrays::varbinview::VarBinViewData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::variant::VariantData - -pub fn vortex_array::arrays::variant::VariantData::into_array(self) -> vortex_array::ArrayRef - impl vortex_array::IntoArray for vortex_buffer::bit::buf::BitBuffer pub fn vortex_buffer::bit::buf::BitBuffer::into_array(self) -> vortex_array::ArrayRef @@ -25578,9 +23964,7 @@ pub fn vortex_array::ArrayRef::to_varbinview(&self) -> vortex_array::arrays::Var pub trait vortex_array::VTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug -pub type vortex_array::VTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug + vortex_array::IntoArray - -pub type vortex_array::VTable::Metadata: core::fmt::Debug +pub type vortex_array::VTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug pub type vortex_array::VTable::OperationsVTable: vortex_array::OperationsVTable @@ -25596,15 +23980,11 @@ pub fn vortex_array::VTable::buffer(array: vortex_array::ArrayView<'_, Self>, id pub fn vortex_array::VTable::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::VTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::VTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::VTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::VTable::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::VTable::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::VTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::VTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -25612,10 +23992,6 @@ pub fn vortex_array::VTable::execute_parent(array: vortex_array::ArrayView<'_, S pub fn vortex_array::VTable::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::VTable::len(array: &Self::ArrayData) -> usize - -pub fn vortex_array::VTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::VTable::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::VTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25624,15 +24000,13 @@ pub fn vortex_array::VTable::reduce(array: vortex_array::ArrayView<'_, Self>) -> pub fn vortex_array::VTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::VTable::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::VTable::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::VTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::VTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] -pub fn vortex_array::VTable::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::VTable::vtable(array: &Self::ArrayData) -> &Self +pub fn vortex_array::VTable::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::VTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25640,8 +24014,6 @@ impl vortex_array::VTable for vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ArrayData = vortex_array::arrays::bool::BoolData -pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Bool::OperationsVTable = vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool @@ -25656,15 +24028,11 @@ pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Bool::dtype(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -25672,10 +24040,6 @@ pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Bool::len(array: &vortex_array::arrays::bool::BoolData) -> usize - -pub fn vortex_array::arrays::Bool::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Bool::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25684,15 +24048,13 @@ pub fn vortex_array::arrays::Bool::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Bool::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25700,8 +24062,6 @@ impl vortex_array::VTable for vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ArrayData = vortex_array::arrays::chunked::ChunkedData -pub type vortex_array::arrays::Chunked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Chunked::OperationsVTable = vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::Chunked @@ -25716,15 +24076,11 @@ pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Chunked::dtype(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -25732,10 +24088,6 @@ pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayV pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Chunked::len(array: &vortex_array::arrays::chunked::ChunkedData) -> usize - -pub fn vortex_array::arrays::Chunked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Chunked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25744,15 +24096,13 @@ pub fn vortex_array::arrays::Chunked::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Chunked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25760,8 +24110,6 @@ impl vortex_array::VTable for vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ArrayData = vortex_array::arrays::constant::ConstantData -pub type vortex_array::arrays::Constant::Metadata = vortex_array::scalar::Scalar - pub type vortex_array::arrays::Constant::OperationsVTable = vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays::Constant @@ -25776,15 +24124,11 @@ pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Constant::dtype(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -25792,10 +24136,6 @@ pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::Array pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Constant::len(array: &vortex_array::arrays::constant::ConstantData) -> usize - -pub fn vortex_array::arrays::Constant::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Constant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25804,15 +24144,13 @@ pub fn vortex_array::arrays::Constant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Constant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25820,8 +24158,6 @@ impl vortex_array::VTable for vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ArrayData = vortex_array::arrays::decimal::DecimalData -pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Decimal::OperationsVTable = vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::Decimal @@ -25836,15 +24172,11 @@ pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Decimal::dtype(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -25852,10 +24184,6 @@ pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayV pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Decimal::len(array: &vortex_array::arrays::decimal::DecimalData) -> usize - -pub fn vortex_array::arrays::Decimal::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Decimal::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25864,15 +24192,13 @@ pub fn vortex_array::arrays::Decimal::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25880,8 +24206,6 @@ impl vortex_array::VTable for vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData -pub type vortex_array::arrays::Extension::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -25896,15 +24220,11 @@ pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Extension::dtype(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -25912,10 +24232,6 @@ pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::Arra pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Extension::len(array: &vortex_array::arrays::extension::ExtensionData) -> usize - -pub fn vortex_array::arrays::Extension::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Extension::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25924,15 +24240,13 @@ pub fn vortex_array::arrays::Extension::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Extension::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Extension::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25940,8 +24254,6 @@ impl vortex_array::VTable for vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ArrayData = vortex_array::arrays::filter::FilterData -pub type vortex_array::arrays::Filter::Metadata = vortex_array::arrays::filter::vtable::FilterMetadata - pub type vortex_array::arrays::Filter::OperationsVTable = vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Filter @@ -25956,15 +24268,11 @@ pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::filter::vtable::FilterMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Filter::dtype(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -25972,10 +24280,6 @@ pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Filter::len(array: &vortex_array::arrays::filter::FilterData) -> usize - -pub fn vortex_array::arrays::Filter::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Filter::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25984,15 +24288,13 @@ pub fn vortex_array::arrays::Filter::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Filter::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Filter::vtable(_array: &vortex_array::arrays::filter::FilterData) -> &Self +pub fn vortex_array::arrays::Filter::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26000,8 +24302,6 @@ impl vortex_array::VTable for vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ArrayData = vortex_array::arrays::fixed_size_list::FixedSizeListData -pub type vortex_array::arrays::FixedSizeList::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::FixedSizeList::OperationsVTable = vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arrays::FixedSizeList @@ -26016,15 +24316,11 @@ pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::FixedSizeList::dtype(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -26032,10 +24328,6 @@ pub fn vortex_array::arrays::FixedSizeList::execute_parent(array: vortex_array:: pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::FixedSizeList::len(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> usize - -pub fn vortex_array::arrays::FixedSizeList::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::FixedSizeList::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26044,15 +24336,13 @@ pub fn vortex_array::arrays::FixedSizeList::reduce(array: vortex_array::ArrayVie pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::FixedSizeList::vtable(_array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &Self +pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26060,8 +24350,6 @@ impl vortex_array::VTable for vortex_array::arrays::List pub type vortex_array::arrays::List::ArrayData = vortex_array::arrays::list::ListData -pub type vortex_array::arrays::List::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::List::OperationsVTable = vortex_array::arrays::List pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List @@ -26076,15 +24364,11 @@ pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::List::dtype(array: &vortex_array::arrays::list::ListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -26092,10 +24376,6 @@ pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::List::len(array: &vortex_array::arrays::list::ListData) -> usize - -pub fn vortex_array::arrays::List::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::List::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26104,15 +24384,13 @@ pub fn vortex_array::arrays::List::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::List::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::list::ListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::List::vtable(_array: &vortex_array::arrays::list::ListData) -> &Self +pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26120,8 +24398,6 @@ impl vortex_array::VTable for vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ArrayData = vortex_array::arrays::listview::ListViewData -pub type vortex_array::arrays::ListView::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::ListView::OperationsVTable = vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays::ListView @@ -26136,15 +24412,11 @@ pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::ListView::dtype(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -26152,10 +24424,6 @@ pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::Array pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::ListView::len(array: &vortex_array::arrays::listview::ListViewData) -> usize - -pub fn vortex_array::arrays::ListView::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::ListView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26164,15 +24432,13 @@ pub fn vortex_array::arrays::ListView::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::ListView::vtable(_array: &vortex_array::arrays::listview::ListViewData) -> &Self +pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26180,8 +24446,6 @@ impl vortex_array::VTable for vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ArrayData = vortex_array::arrays::masked::MaskedData -pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked @@ -26196,15 +24460,11 @@ pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Masked::dtype(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -26212,10 +24472,6 @@ pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Masked::len(array: &vortex_array::arrays::masked::MaskedData) -> usize - -pub fn vortex_array::arrays::Masked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Masked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26224,15 +24480,13 @@ pub fn vortex_array::arrays::Masked::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Masked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Masked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26240,8 +24494,6 @@ impl vortex_array::VTable for vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ArrayData = vortex_array::arrays::primitive::PrimitiveData -pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive @@ -26256,15 +24508,11 @@ pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Primitive::dtype(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -26272,10 +24520,6 @@ pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::Arra pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Primitive::len(array: &vortex_array::arrays::primitive::PrimitiveData) -> usize - -pub fn vortex_array::arrays::Primitive::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Primitive::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26284,15 +24528,13 @@ pub fn vortex_array::arrays::Primitive::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26300,8 +24542,6 @@ impl vortex_array::VTable for vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ArrayData = vortex_array::arrays::shared::SharedData -pub type vortex_array::arrays::Shared::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Shared::OperationsVTable = vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Shared @@ -26316,15 +24556,11 @@ pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Shared::dtype(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -26332,10 +24568,6 @@ pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Shared::len(array: &vortex_array::arrays::shared::SharedData) -> usize - -pub fn vortex_array::arrays::Shared::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Shared::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26344,15 +24576,13 @@ pub fn vortex_array::arrays::Shared::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Shared::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Shared::vtable(_array: &vortex_array::arrays::shared::SharedData) -> &Self +pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26360,8 +24590,6 @@ impl vortex_array::VTable for vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ArrayData = vortex_array::arrays::struct_::StructData -pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct @@ -26376,15 +24604,11 @@ pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Struct::dtype(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -26392,10 +24616,6 @@ pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Struct::len(array: &vortex_array::arrays::struct_::StructData) -> usize - -pub fn vortex_array::arrays::Struct::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Struct::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26404,15 +24624,13 @@ pub fn vortex_array::arrays::Struct::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Struct::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Struct::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26420,8 +24638,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ArrayData = vortex_array::arrays::varbin::VarBinData -pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::VarBin::OperationsVTable = vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::VarBin @@ -26436,15 +24652,11 @@ pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBin::dtype(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -26452,10 +24664,6 @@ pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayVi pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::VarBin::len(array: &vortex_array::arrays::varbin::VarBinData) -> usize - -pub fn vortex_array::arrays::VarBin::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBin::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26464,15 +24672,13 @@ pub fn vortex_array::arrays::VarBin::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBin::vtable(_array: &vortex_array::arrays::varbin::VarBinData) -> &Self +pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26480,8 +24686,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ArrayData = vortex_array::arrays::varbinview::VarBinViewData -pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView @@ -26496,15 +24700,11 @@ pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBinView::dtype(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -26512,10 +24712,6 @@ pub fn vortex_array::arrays::VarBinView::execute_parent(array: vortex_array::Arr pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::VarBinView::len(array: &vortex_array::arrays::varbinview::VarBinViewData) -> usize - -pub fn vortex_array::arrays::VarBinView::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBinView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26524,15 +24720,13 @@ pub fn vortex_array::arrays::VarBinView::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBinView::vtable(_array: &vortex_array::arrays::varbinview::VarBinViewData) -> &Self +pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26540,8 +24734,6 @@ impl vortex_array::VTable for vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ArrayData = vortex_array::arrays::variant::VariantData -pub type vortex_array::arrays::Variant::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Variant::OperationsVTable = vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::Variant @@ -26556,15 +24748,11 @@ pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Variant::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -26572,10 +24760,6 @@ pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayV pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::Variant::len(array: &Self::ArrayData) -> usize - -pub fn vortex_array::arrays::Variant::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Variant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26584,15 +24768,13 @@ pub fn vortex_array::arrays::Variant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Variant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Variant::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26600,8 +24782,6 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData -pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -26616,15 +24796,11 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -26632,10 +24808,6 @@ pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::Arr pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize - -pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::dict::Dict::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26644,15 +24816,13 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26660,8 +24830,6 @@ impl vortex_array::VTable for vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ArrayData = vortex_array::arrays::null::NullData -pub type vortex_array::arrays::null::Null::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::null::Null::OperationsVTable = vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays::null::Null @@ -26676,15 +24844,11 @@ pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::null::Null::dtype(_array: &vortex_array::arrays::null::NullData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -26692,10 +24856,6 @@ pub fn vortex_array::arrays::null::Null::execute_parent(array: vortex_array::Arr pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::null::Null::len(array: &vortex_array::arrays::null::NullData) -> usize - -pub fn vortex_array::arrays::null::Null::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::null::Null::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26704,15 +24864,13 @@ pub fn vortex_array::arrays::null::Null::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::null::Null::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::null::Null::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26720,8 +24878,6 @@ impl vortex_array::VTable for vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray -pub type vortex_array::arrays::patched::Patched::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -26736,15 +24892,11 @@ pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::patched::Patched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -26752,10 +24904,6 @@ pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_arra pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::patched::Patched::len(array: &Self::ArrayData) -> usize - -pub fn vortex_array::arrays::patched::Patched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::patched::Patched::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26764,15 +24912,13 @@ pub fn vortex_array::arrays::patched::Patched::reduce(array: vortex_array::Array pub fn vortex_array::arrays::patched::Patched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::patched::Patched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::patched::Patched::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26780,8 +24926,6 @@ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::Metadata = vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata - pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -26796,15 +24940,11 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::dtype(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -26812,10 +24952,6 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vo pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::len(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> usize - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26824,15 +24960,13 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_arr pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::vtable(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &Self +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26840,8 +24974,6 @@ impl vortex_array::VTable for vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ArrayData = vortex_array::arrays::slice::SliceData -pub type vortex_array::arrays::slice::Slice::Metadata = vortex_array::arrays::slice::SliceMetadata - pub type vortex_array::arrays::slice::Slice::OperationsVTable = vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arrays::slice::Slice @@ -26856,15 +24988,11 @@ pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::slice::SliceMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -26872,10 +25000,6 @@ pub fn vortex_array::arrays::slice::Slice::execute_parent(array: vortex_array::A pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::slice::Slice::len(array: &vortex_array::arrays::slice::SliceData) -> usize - -pub fn vortex_array::arrays::slice::Slice::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::slice::Slice::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26884,15 +25008,13 @@ pub fn vortex_array::arrays::slice::Slice::reduce(array: vortex_array::ArrayView pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::slice::Slice::vtable(_array: &vortex_array::arrays::slice::SliceData) -> &Self +pub fn vortex_array::arrays::slice::Slice::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> diff --git a/vortex-array/src/array/typed.rs b/vortex-array/src/array/typed.rs index 8f1d1b50f49..85a6fb56fc5 100644 --- a/vortex-array/src/array/typed.rs +++ b/vortex-array/src/array/typed.rs @@ -25,7 +25,7 @@ use crate::stats::ArrayStats; use crate::stats::StatsSetRef; use crate::validity::Validity; -/// Construction parameters for [`ArrayInner`] / [`Array`]. +/// Construction parameters for typed arrays. pub struct ArrayParts { pub vtable: V, pub dtype: DType, @@ -245,6 +245,18 @@ impl Array { &self.downcast_inner().data } + /// Returns the full typed array construction parts. + pub fn into_parts(self) -> ArrayParts { + let inner = self.downcast_inner(); + ArrayParts { + vtable: inner.vtable.clone(), + dtype: inner.dtype.clone(), + len: inner.len, + data: inner.data.clone(), + stats: inner.stats.clone(), + } + } + /// Returns a clone of the inner encoding-specific data. pub fn into_data(self) -> V::ArrayData { self.downcast_inner().data.clone() @@ -424,3 +436,25 @@ impl From> for ArrayRef { value.inner } } + +#[cfg(test)] +mod tests { + use vortex_buffer::buffer; + + use super::Array; + use crate::assert_arrays_eq; + use crate::arrays::Primitive; + use crate::arrays::PrimitiveArray; + use crate::validity::Validity; + + #[test] + fn typed_array_into_parts_roundtrips() { + let array = PrimitiveArray::new(buffer![1i32, 2, 3], Validity::NonNullable); + let expected = array.clone(); + + let parts = array.into_parts(); + let rebuilt = Array::::try_from_parts(parts).unwrap(); + + assert_arrays_eq!(rebuilt, expected); + } +} diff --git a/vortex-array/src/arrays/bool/array.rs b/vortex-array/src/arrays/bool/array.rs index 3f32ecbae50..db22f04ecf1 100644 --- a/vortex-array/src/arrays/bool/array.rs +++ b/vortex-array/src/arrays/bool/array.rs @@ -242,9 +242,9 @@ impl Array { self.into_data().into_bit_buffer() } - /// Splits into owned parts, consuming self. + /// Splits into the encoding-specific owned parts, consuming self. #[inline] - pub fn into_parts(self) -> BoolArrayParts { + pub fn into_encoding_parts(self) -> BoolArrayParts { self.into_data().into_parts() } } diff --git a/vortex-array/src/arrays/chunked/array.rs b/vortex-array/src/arrays/chunked/array.rs index 0d865e8846d..857eb43c777 100644 --- a/vortex-array/src/arrays/chunked/array.rs +++ b/vortex-array/src/arrays/chunked/array.rs @@ -34,7 +34,7 @@ pub(super) const CHUNKS_OFFSET: usize = 1; #[derive(Clone, Debug)] pub struct ChunkedData { - pub(super) dtype: DType, + pub(super) empty_dtype: Option, pub(super) chunk_offsets: PrimitiveData, pub(super) chunks: Vec, pub(super) slots: Vec>, @@ -105,7 +105,7 @@ impl ChunkedData { let slots = Self::make_slots(&chunk_offsets, &chunks); Self { - dtype, + empty_dtype: chunks.is_empty().then_some(dtype), chunk_offsets, chunks, slots, @@ -136,7 +136,11 @@ impl ChunkedData { /// Returns the [`DType`] of this array. pub fn dtype(&self) -> &DType { - &self.dtype + self.chunks + .first() + .map(|chunk| chunk.dtype()) + .or(self.empty_dtype.as_ref()) + .vortex_expect("ChunkedArray dtype must come from chunks or the empty dtype") } /// Returns `true` if this array is empty. @@ -274,6 +278,16 @@ impl Array { Array::try_from_parts(ArrayParts::new(Chunked, dtype, len, data)) } + pub fn rechunk(&self, target_bytesize: u64, target_rowsize: usize) -> VortexResult { + let data = self.data().rechunk(target_bytesize, target_rowsize)?; + Array::try_from_parts(ArrayParts::new( + Chunked, + self.dtype().clone(), + data.len(), + data, + )) + } + /// Creates a new `ChunkedArray` without validation. /// /// # Safety diff --git a/vortex-array/src/arrays/chunked/tests.rs b/vortex-array/src/arrays/chunked/tests.rs index 8eefe189a29..f1266a5373d 100644 --- a/vortex-array/src/arrays/chunked/tests.rs +++ b/vortex-array/src/arrays/chunked/tests.rs @@ -8,6 +8,7 @@ use vortex_buffer::buffer; use crate::IntoArray; use crate::accessor::ArrayAccessor; +use crate::array::ArrayParts; use crate::array::VTable; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; @@ -207,10 +208,12 @@ fn with_slots_updates_nchunks_len_and_offsets() { ]; let expected_nchunks = slots.len() - 1; let expected_len = orig.len(); + let dtype = orig.dtype().clone(); let mut data = orig.into_data(); ::with_slots(&mut data, slots).unwrap(); - let array = ChunkedArray::try_from_data(data).unwrap(); + let array = + ChunkedArray::try_from_parts(ArrayParts::new(Chunked, dtype, expected_len, data)).unwrap(); assert_eq!(array.nchunks(), expected_nchunks); assert_eq!(array.len(), expected_len); diff --git a/vortex-array/src/arrays/chunked/vtable/mod.rs b/vortex-array/src/arrays/chunked/vtable/mod.rs index 30e5dd49232..0c9aaca6ef2 100644 --- a/vortex-array/src/arrays/chunked/vtable/mod.rs +++ b/vortex-array/src/arrays/chunked/vtable/mod.rs @@ -62,26 +62,31 @@ impl VTable for Chunked { } fn array_hash(array: &ChunkedData, state: &mut H, precision: Precision) { - PrimitiveArray::new(array.chunk_offsets.to_buffer::(), Validity::NonNullable) - .into_array() - .array_hash(state, precision); + PrimitiveArray::new( + array.chunk_offsets.to_buffer::(), + Validity::NonNullable, + ) + .into_array() + .array_hash(state, precision); for chunk in &array.chunks { chunk.array_hash(state, precision); } } fn array_eq(array: &ChunkedData, other: &ChunkedData, precision: Precision) -> bool { - PrimitiveArray::new(array.chunk_offsets.to_buffer::(), Validity::NonNullable) - .into_array() - .array_eq( - &PrimitiveArray::new( - other.chunk_offsets.to_buffer::(), - Validity::NonNullable, - ) - .into_array(), - precision, + PrimitiveArray::new( + array.chunk_offsets.to_buffer::(), + Validity::NonNullable, + ) + .into_array() + .array_eq( + &PrimitiveArray::new( + other.chunk_offsets.to_buffer::(), + Validity::NonNullable, ) - && array.chunks.len() == other.chunks.len() + .into_array(), + precision, + ) && array.chunks.len() == other.chunks.len() && array .iter_chunks() .zip(other.iter_chunks()) @@ -123,7 +128,8 @@ impl VTable for Chunked { fn deserialize( &self, dtype: &DType, - _len: usize, metadata: &[u8], + _len: usize, + metadata: &[u8], _buffers: &[BufferHandle], children: &dyn ArrayChildren, @@ -165,7 +171,7 @@ impl VTable for Chunked { let slots = ChunkedData::make_slots(&chunk_offsets, &chunks); // Construct directly using the struct fields to avoid recomputing chunk_offsets Ok(ChunkedData { - dtype: dtype.clone(), + empty_dtype: chunks.is_empty().then_some(dtype.clone()), chunk_offsets, chunks, slots, @@ -208,6 +214,7 @@ impl VTable for Chunked { }) .try_collect()?; array.chunk_offsets = PrimitiveData::new(chunk_offsets_buf.clone(), Validity::NonNullable); + array.empty_dtype = chunks.is_empty().then_some(array.dtype().clone()); array.chunks = chunks; array.slots = slots; diff --git a/vortex-array/src/arrays/constant/array.rs b/vortex-array/src/arrays/constant/array.rs index e4b3bb7b20e..5514221d379 100644 --- a/vortex-array/src/arrays/constant/array.rs +++ b/vortex-array/src/arrays/constant/array.rs @@ -23,7 +23,10 @@ impl ConstantData { S: Into, { let scalar = scalar.into(); - Self { scalar, slots: vec![] } + Self { + scalar, + slots: vec![], + } } /// Returns the [`Scalar`] value of this constant array. diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index fc61e0311f2..d8694634ce5 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -27,8 +27,8 @@ use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; use crate::array::VTable; -use crate::arrays::Primitive; use crate::arrays::ConstantArray; +use crate::arrays::Primitive; use crate::arrays::dict::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -97,18 +97,20 @@ impl VTable for Dict { } fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some(ProstMetadata(DictMetadata { - codes_ptype: PType::try_from(array.codes().dtype())? as i32, - values_len: u32::try_from(array.values().len()).map_err(|_| { - vortex_err!( - "Dictionary values size {} overflowed u32", - array.values().len() - ) - })?, - is_nullable_codes: Some(array.codes().dtype().is_nullable()), - all_values_referenced: Some(array.all_values_referenced), - }) - .serialize())) + Ok(Some( + ProstMetadata(DictMetadata { + codes_ptype: PType::try_from(array.codes().dtype())? as i32, + values_len: u32::try_from(array.values().len()).map_err(|_| { + vortex_err!( + "Dictionary values size {} overflowed u32", + array.values().len() + ) + })?, + is_nullable_codes: Some(array.codes().dtype().is_nullable()), + all_values_referenced: Some(array.all_values_referenced), + }) + .serialize(), + )) } fn deserialize( diff --git a/vortex-array/src/arrays/dict/vtable/validity.rs b/vortex-array/src/arrays/dict/vtable/validity.rs index e9c51c4b53e..fc13dcdc84d 100644 --- a/vortex-array/src/arrays/dict/vtable/validity.rs +++ b/vortex-array/src/arrays/dict/vtable/validity.rs @@ -31,15 +31,15 @@ impl ValidityVTable for Dict { (Validity::AllValid | Validity::NonNullable, Validity::Array(values_validity)) => { // We know codes are all valid, so the cast is free. let codes = array.codes().cast(array.codes().dtype().as_nonnullable())?; - Validity::Array(unsafe { DictArray::new_unchecked(codes, values_validity) } - .into_array()) + Validity::Array( + unsafe { DictArray::new_unchecked(codes, values_validity) }.into_array(), + ) } (Validity::Array(_codes_validity), Validity::Array(values_validity)) => { // Create a mask representing "is the value at codes[i] valid?" - let values_valid_mask = unsafe { - DictArray::new_unchecked(array.codes().clone(), values_validity) - } - .into_array(); + let values_valid_mask = + unsafe { DictArray::new_unchecked(array.codes().clone(), values_validity) } + .into_array(); let values_valid_mask = values_valid_mask .fill_null(Scalar::bool(false, Nullability::NonNullable))?; diff --git a/vortex-array/src/arrays/extension/compute/cast.rs b/vortex-array/src/arrays/extension/compute/cast.rs index 161592ad264..31ceb8ccf8f 100644 --- a/vortex-array/src/arrays/extension/compute/cast.rs +++ b/vortex-array/src/arrays/extension/compute/cast.rs @@ -79,7 +79,7 @@ mod tests { let storage = Buffer::::empty().into_array(); let arr = ExtensionArray::new(ext_dtype.clone(), storage); - assert!(!arr.dtype.is_nullable()); + assert!(!arr.dtype().is_nullable()); let new_dtype = DType::Extension(ext_dtype).with_nullability(Nullability::Nullable); diff --git a/vortex-array/src/arrays/extension/vtable/mod.rs b/vortex-array/src/arrays/extension/vtable/mod.rs index 4335b88a491..c888da35eee 100644 --- a/vortex-array/src/arrays/extension/vtable/mod.rs +++ b/vortex-array/src/arrays/extension/vtable/mod.rs @@ -105,7 +105,8 @@ impl VTable for Extension { fn deserialize( &self, dtype: &DType, - len: usize, metadata: &[u8], + len: usize, + metadata: &[u8], _buffers: &[BufferHandle], children: &dyn ArrayChildren, diff --git a/vortex-array/src/arrays/fixed_size_list/array.rs b/vortex-array/src/arrays/fixed_size_list/array.rs index f5bdec71787..052f38bb127 100644 --- a/vortex-array/src/arrays/fixed_size_list/array.rs +++ b/vortex-array/src/arrays/fixed_size_list/array.rs @@ -100,7 +100,6 @@ pub struct FixedSizeListData { /// and `elements.len()`) is because in the degenerate case where `list_size == 0`, we cannot /// use `0 / 0` to determine the length. pub(super) degenerate_len: usize, - } impl FixedSizeListData { @@ -173,13 +172,14 @@ impl FixedSizeListData { } pub fn into_parts(mut self) -> (ArrayRef, Validity, DType) { + let dtype = self.dtype(); let validity = self.validity(); ( self.slots[ELEMENTS_SLOT] .take() .vortex_expect("FixedSizeListArray elements slot"), validity, - self.dtype(), + dtype, ) } diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs index 2d0234b169d..59bd04e278b 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs @@ -127,7 +127,8 @@ impl VTable for FixedSizeList { fn deserialize( &self, dtype: &DType, - len: usize, metadata: &[u8], + len: usize, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, diff --git a/vortex-array/src/arrays/list/array.rs b/vortex-array/src/arrays/list/array.rs index 25fe87e145d..6f76fd9917c 100644 --- a/vortex-array/src/arrays/list/array.rs +++ b/vortex-array/src/arrays/list/array.rs @@ -24,8 +24,8 @@ use crate::arrays::List; use crate::arrays::Primitive; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; -use crate::dtype::Nullability; use crate::dtype::NativePType; +use crate::dtype::Nullability; use crate::match_each_integer_ptype; use crate::match_each_native_ptype; use crate::scalar_fn::fns::operators::Operator; diff --git a/vortex-array/src/arrays/list/tests.rs b/vortex-array/src/arrays/list/tests.rs index 874be33275b..f815acb3bb0 100644 --- a/vortex-array/src/arrays/list/tests.rs +++ b/vortex-array/src/arrays/list/tests.rs @@ -14,6 +14,8 @@ use crate::Canonical; use crate::IntoArray; use crate::LEGACY_SESSION; use crate::VortexSessionExecute; +use crate::array::Array; +use crate::array::ArrayParts; use crate::arrays::FilterArray; use crate::arrays::List; use crate::arrays::PrimitiveArray; @@ -913,9 +915,23 @@ fn test_recursive_compact_list_of_lists() { assert_eq!(recursive_flat_elements.len(), 7); // Verify data integrity is preserved + let non_recursive_array = Array::try_from_parts(ArrayParts::new( + List, + non_recursive.dtype(), + non_recursive.len(), + non_recursive.clone(), + )) + .unwrap(); + let recursive_array = Array::try_from_parts(ArrayParts::new( + List, + recursive.dtype(), + recursive.len(), + recursive.clone(), + )) + .unwrap(); assert_eq!( - non_recursive.clone().into_array().scalar_at(0).unwrap(), - recursive.clone().into_array().scalar_at(0).unwrap() + non_recursive_array.scalar_at(0).unwrap(), + recursive_array.scalar_at(0).unwrap() ); } diff --git a/vortex-array/src/arrays/list/vtable/mod.rs b/vortex-array/src/arrays/list/vtable/mod.rs index 52683a04e43..fb267eff690 100644 --- a/vortex-array/src/arrays/list/vtable/mod.rs +++ b/vortex-array/src/arrays/list/vtable/mod.rs @@ -89,11 +89,13 @@ impl VTable for List { } fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some(ProstMetadata(ListMetadata { - elements_len: array.elements().len() as u64, - offset_ptype: PType::try_from(array.offsets().dtype())? as i32, - }) - .serialize())) + Ok(Some( + ProstMetadata(ListMetadata { + elements_len: array.elements().len() as u64, + offset_ptype: PType::try_from(array.offsets().dtype())? as i32, + }) + .serialize(), + )) } fn validate(&self, data: &ListData, dtype: &DType, len: usize) -> VortexResult<()> { @@ -118,7 +120,8 @@ impl VTable for List { fn deserialize( &self, dtype: &DType, - len: usize, metadata: &[u8], + len: usize, + metadata: &[u8], _buffers: &[BufferHandle], children: &dyn ArrayChildren, diff --git a/vortex-array/src/arrays/listview/array.rs b/vortex-array/src/arrays/listview/array.rs index eb3fc9d6ccf..b4a3eca0181 100644 --- a/vortex-array/src/arrays/listview/array.rs +++ b/vortex-array/src/arrays/listview/array.rs @@ -123,7 +123,6 @@ pub struct ListViewData { /// `offsets[i] + sizes[i]` are in order), conversions can bypass the very expensive rebuild /// process which must rebuild the array from scratch. is_zero_copy_to_list: bool, - } pub struct ListViewArrayParts { diff --git a/vortex-array/src/arrays/listview/vtable/mod.rs b/vortex-array/src/arrays/listview/vtable/mod.rs index cef458ce078..cc87352ec65 100644 --- a/vortex-array/src/arrays/listview/vtable/mod.rs +++ b/vortex-array/src/arrays/listview/vtable/mod.rs @@ -89,12 +89,14 @@ impl VTable for ListView { } fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some(ProstMetadata(ListViewMetadata { - elements_len: array.elements().len() as u64, - offset_ptype: PType::try_from(array.offsets().dtype())? as i32, - size_ptype: PType::try_from(array.sizes().dtype())? as i32, - }) - .serialize())) + Ok(Some( + ProstMetadata(ListViewMetadata { + elements_len: array.elements().len() as u64, + offset_ptype: PType::try_from(array.offsets().dtype())? as i32, + size_ptype: PType::try_from(array.sizes().dtype())? as i32, + }) + .serialize(), + )) } fn validate(&self, data: &ListViewData, dtype: &DType, len: usize) -> VortexResult<()> { @@ -119,7 +121,8 @@ impl VTable for ListView { fn deserialize( &self, dtype: &DType, - len: usize, metadata: &[u8], + len: usize, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, diff --git a/vortex-array/src/arrays/masked/compute/filter.rs b/vortex-array/src/arrays/masked/compute/filter.rs index b810248b614..0eb8fb25011 100644 --- a/vortex-array/src/arrays/masked/compute/filter.rs +++ b/vortex-array/src/arrays/masked/compute/filter.rs @@ -21,7 +21,9 @@ impl FilterReduce for Masked { let filtered_child = array.child().filter(mask.clone())?; // Construct new MaskedArray - Ok(Some(MaskedArray::try_new(filtered_child, filtered_validity)?.into_array())) + Ok(Some( + MaskedArray::try_new(filtered_child, filtered_validity)?.into_array(), + )) } } diff --git a/vortex-array/src/arrays/masked/compute/take.rs b/vortex-array/src/arrays/masked/compute/take.rs index cfe1653ea4c..7061529b897 100644 --- a/vortex-array/src/arrays/masked/compute/take.rs +++ b/vortex-array/src/arrays/masked/compute/take.rs @@ -27,7 +27,9 @@ impl TakeReduce for Masked { let taken_validity = array.validity().take(indices)?; // Construct new MaskedArray - Ok(Some(MaskedArray::try_new(taken_child, taken_validity)?.into_array())) + Ok(Some( + MaskedArray::try_new(taken_child, taken_validity)?.into_array(), + )) } } diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index 269fb626a92..dadd023e3d2 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -55,7 +55,10 @@ impl VTable for Masked { } fn validate(&self, data: &MaskedData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!(data.child().len() == len, "MaskedArray child length mismatch"); + vortex_ensure!( + data.child().len() == len, + "MaskedArray child length mismatch" + ); vortex_ensure!( data.dtype() == *dtype, "MaskedArray dtype does not match child and validity" diff --git a/vortex-array/src/arrays/patched/array.rs b/vortex-array/src/arrays/patched/array.rs index 98460c46a6c..a9913b2c118 100644 --- a/vortex-array/src/arrays/patched/array.rs +++ b/vortex-array/src/arrays/patched/array.rs @@ -27,7 +27,6 @@ use crate::dtype::PType; use crate::match_each_native_ptype; use crate::match_each_unsigned_integer_ptype; use crate::patches::Patches; -use crate::stats::ArrayStats; use crate::validity::Validity; /// An array that partially "patches" another array with new values. @@ -127,22 +126,15 @@ pub struct PatchedArray { /// should be subtracted out of the remaining offsets to get their final position in the /// executed array. pub(super) offset: usize, - /// Length of the array - pub(super) len: usize, - - pub(super) stats_set: ArrayStats, } impl IntoArray for PatchedArray { fn into_array(self) -> ArrayRef { let dtype = self.base_array().dtype().clone(); - let len = self.len; - let stats = self.stats_set.clone(); - Array::::try_from_parts( - ArrayParts::new(Patched, dtype, len, self).with_stats(stats), - ) - .vortex_expect("PatchedArray is always valid") - .into_array() + let len = self.len(); + Array::::try_from_parts(ArrayParts::new(Patched, dtype, len, self)) + .vortex_expect("PatchedArray is always valid") + .into_array() } } @@ -214,14 +206,10 @@ impl PatchedArray { ) .into_array(); - let len = inner.len(); - Ok(Self { slots: vec![Some(inner), Some(lane_offsets), Some(indices), Some(values)], n_lanes, offset: 0, - len, - stats_set: ArrayStats::default(), }) } } @@ -258,6 +246,37 @@ impl PatchedArray { .as_ref() .vortex_expect("PatchedArray values slot") } + + #[inline] + pub fn len(&self) -> usize { + self.base_array().len() + } + + pub fn validate_against_outer( + &self, + dtype: &crate::dtype::DType, + len: usize, + ) -> VortexResult<()> { + vortex_ensure!( + self.base_array().dtype() == dtype, + "PatchedArray base dtype {} does not match outer dtype {}", + self.base_array().dtype(), + dtype + ); + vortex_ensure!( + self.len() == len, + "PatchedArray base len {} does not match outer len {}", + self.len(), + len + ); + vortex_ensure!( + self.patch_indices().len() == self.patch_values().len(), + "PatchedArray patch indices len {} does not match patch values len {}", + self.patch_indices().len(), + self.patch_values().len() + ); + Ok(()) + } } impl PatchedArray { @@ -268,7 +287,7 @@ impl PatchedArray { /// /// Note that this function will panic if the caller requests out of bounds chunk/lane ordinals. pub(crate) fn lane_range(&self, chunk: usize, lane: usize) -> VortexResult> { - assert!(chunk * 1024 <= self.len + self.offset); + assert!(chunk * 1024 <= self.len() + self.offset); assert!(lane < self.n_lanes); let start = self.lane_offsets().scalar_at(chunk * self.n_lanes + lane)?; @@ -306,14 +325,12 @@ impl PatchedArray { let begin = (chunks.start * 1024).saturating_sub(self.offset); let end = (chunks.end * 1024) .saturating_sub(self.offset) - .min(self.len); + .min(self.len()); let offset = if chunks.start == 0 { self.offset } else { 0 }; let inner = self.base_array().slice(begin..end)?; - let len = end - begin; - Ok(PatchedArray { slots: vec![ Some(inner), @@ -323,8 +340,6 @@ impl PatchedArray { ], n_lanes: self.n_lanes, offset, - len, - stats_set: ArrayStats::default(), }) } } diff --git a/vortex-array/src/arrays/patched/compute/compare.rs b/vortex-array/src/arrays/patched/compute/compare.rs index 0ec4448da37..0af82fcf7b3 100644 --- a/vortex-array/src/arrays/patched/compute/compare.rs +++ b/vortex-array/src/arrays/patched/compute/compare.rs @@ -55,7 +55,7 @@ impl CompareKernel for Patched { offset, len, validity, - } = result.into_parts(); + } = result.into_encoding_parts(); let mut bits = BitBufferMut::from_buffer(bits.unwrap_host().into_mut(), offset, len); @@ -164,7 +164,6 @@ mod tests { use crate::ExecutionCtx; use crate::IntoArray; use crate::LEGACY_SESSION; - use crate::array::Array; use crate::arrays::BoolArray; use crate::arrays::ConstantArray; use crate::arrays::Patched; @@ -191,10 +190,11 @@ mod tests { let mut ctx = ExecutionCtx::new(LEGACY_SESSION.clone()); - let lhs = Array::::try_from_data( - PatchedArray::from_array_and_patches(lhs, &patches, &mut ctx).unwrap(), - ) - .unwrap(); + let lhs = PatchedArray::from_array_and_patches(lhs, &patches, &mut ctx) + .unwrap() + .into_array() + .try_into::() + .unwrap(); let rhs = ConstantArray::new(u32::MAX, 512).into_array(); @@ -259,9 +259,10 @@ mod tests { )?; let mut ctx = ExecutionCtx::new(LEGACY_SESSION.clone()); - let lhs = Array::::try_from_data(PatchedArray::from_array_and_patches( - lhs, &patches, &mut ctx, - )?)?; + let lhs = PatchedArray::from_array_and_patches(lhs, &patches, &mut ctx)? + .into_array() + .try_into::() + .unwrap(); let rhs = ConstantArray::new(subnormal, 512).into_array(); @@ -292,9 +293,10 @@ mod tests { )?; let mut ctx = ExecutionCtx::new(LEGACY_SESSION.clone()); - let lhs = Array::::try_from_data(PatchedArray::from_array_and_patches( - lhs, &patches, &mut ctx, - )?)?; + let lhs = PatchedArray::from_array_and_patches(lhs, &patches, &mut ctx)? + .into_array() + .try_into::() + .unwrap(); let rhs = ConstantArray::new(0.0f32, 10).into_array(); diff --git a/vortex-array/src/arrays/patched/compute/filter.rs b/vortex-array/src/arrays/patched/compute/filter.rs index 71aee0238ee..0a2060964aa 100644 --- a/vortex-array/src/arrays/patched/compute/filter.rs +++ b/vortex-array/src/arrays/patched/compute/filter.rs @@ -37,7 +37,7 @@ impl FilterReduce for Patched { } }; - let n_chunks = (array.offset + array.len).div_ceil(1024); + let n_chunks = (array.offset + array.len()).div_ceil(1024); // If all chunks already covered, there is nothing to do. if chunk_start == 0 && chunk_stop == n_chunks { diff --git a/vortex-array/src/arrays/patched/compute/take.rs b/vortex-array/src/arrays/patched/compute/take.rs index 6fac5826537..9ab496c953d 100644 --- a/vortex-array/src/arrays/patched/compute/take.rs +++ b/vortex-array/src/arrays/patched/compute/take.rs @@ -63,7 +63,7 @@ impl TakeExecute for Patched { output.as_mut(), indices.as_slice::(), array.offset, - array.len, + array.len(), array.n_lanes, lane_offsets.as_slice::(), patch_indices.as_slice::(), diff --git a/vortex-array/src/arrays/patched/vtable/mod.rs b/vortex-array/src/arrays/patched/vtable/mod.rs index 64a5d300917..f319ccc85e2 100644 --- a/vortex-array/src/arrays/patched/vtable/mod.rs +++ b/vortex-array/src/arrays/patched/vtable/mod.rs @@ -47,7 +47,6 @@ use crate::dtype::NativePType; use crate::dtype::PType; use crate::match_each_native_ptype; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::vtable; vtable!(Patched); @@ -115,6 +114,10 @@ impl VTable for Patched { 0 } + fn validate(&self, data: &PatchedArray, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate_against_outer(dtype, len) + } + fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { vortex_panic!("invalid buffer index for PatchedArray: {idx}"); } @@ -171,8 +174,6 @@ impl VTable for Patched { slots: vec![Some(inner), Some(lane_offsets), Some(indices), Some(values)], n_lanes, offset, - len, - stats_set: ArrayStats::default(), }) } @@ -288,7 +289,7 @@ impl VTable for Patched { let patched_values = match_each_native_ptype!(values.ptype(), |V| { let offset = array.offset; - let len = array.len; + let len = array.len(); let mut output = Buffer::::from_byte_buffer(buffer.unwrap_host()).into_mut(); diff --git a/vortex-array/src/arrays/patched/vtable/slice.rs b/vortex-array/src/arrays/patched/vtable/slice.rs index 71d68585ebe..bbdb4df2464 100644 --- a/vortex-array/src/arrays/patched/vtable/slice.rs +++ b/vortex-array/src/arrays/patched/vtable/slice.rs @@ -11,15 +11,12 @@ use crate::array::ArrayView; use crate::arrays::Patched; use crate::arrays::PatchedArray; use crate::arrays::slice::SliceReduce; -use crate::stats::ArrayStats; impl SliceReduce for Patched { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { // We **always** slice the patches at 1024-element chunk boundaries. We keep the offset + len // around so that when we execute we know how much to chop off. let new_offset = (range.start + array.offset) % 1024; - let new_len = range.end - range.start; - let chunk_start = (range.start + array.offset) / 1024; let chunk_stop = (range.end + array.offset).div_ceil(1024); let sliced_lane_offsets = array @@ -41,8 +38,6 @@ impl SliceReduce for Patched { ], n_lanes: array.n_lanes, offset: new_offset, - len: new_len, - stats_set: ArrayStats::default(), } .into_array(), )) diff --git a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs index 9e042fa647d..f3fbbe52894 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs @@ -72,7 +72,10 @@ impl VTable for ScalarFnVTable { "All child arrays must have the same length as the scalar function array" ); - let child_dtypes = data.iter_children().map(|c| c.dtype().clone()).collect_vec(); + let child_dtypes = data + .iter_children() + .map(|c| c.dtype().clone()) + .collect_vec(); vortex_ensure!( self.scalar_fn.return_dtype(&child_dtypes)? == *dtype, "ScalarFnArray dtype does not match scalar function return dtype" diff --git a/vortex-array/src/arrays/struct_/array.rs b/vortex-array/src/arrays/struct_/array.rs index 111bcf286fa..1ec06c97822 100644 --- a/vortex-array/src/arrays/struct_/array.rs +++ b/vortex-array/src/arrays/struct_/array.rs @@ -149,8 +149,9 @@ pub(super) const FIELDS_OFFSET: usize = 1; /// ``` #[derive(Clone, Debug)] pub struct StructData { - pub(super) len: usize, - pub(super) dtype: DType, + pub(super) names: FieldNames, + pub(super) nullability: crate::dtype::Nullability, + pub(super) fieldless_len: Option, pub(super) slots: Vec>, } @@ -163,12 +164,15 @@ pub struct StructArrayParts { impl StructData { /// Returns the length of this array. pub fn len(&self) -> usize { - self.len + self.slots[FIELDS_OFFSET..] + .first() + .and_then(|slot| slot.as_ref()) + .map_or_else(|| self.fieldless_len.unwrap_or(0), |field| field.len()) } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + DType::Struct(self.struct_fields(), self.nullability) } /// Returns `true` if this array is empty. @@ -178,7 +182,7 @@ impl StructData { /// Reconstructs the validity from the slots. pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) } /// Return an iterator over the struct fields without the validity of the struct applied. @@ -214,7 +218,7 @@ impl StructData { /// Return the struct field without the validity of the struct applied pub fn unmasked_field_by_name_opt(&self, name: impl AsRef) -> Option<&ArrayRef> { let name = name.as_ref(); - self.struct_fields().find(name).map(|idx| { + self.names.find(name).map(|idx| { self.slots[FIELDS_OFFSET + idx] .as_ref() .vortex_expect("StructArray field slot") @@ -222,16 +226,16 @@ impl StructData { } pub fn names(&self) -> &FieldNames { - self.struct_fields().names() + &self.names } - pub fn struct_fields(&self) -> &StructFields { - let Some(struct_dtype) = &self.dtype.as_struct_fields_opt() else { - unreachable!( - "struct arrays must have be a DType::Struct, this is likely an internal bug." - ) - }; - struct_dtype + pub fn struct_fields(&self) -> StructFields { + StructFields::new( + self.names.clone(), + self.iter_unmasked_fields() + .map(|field| field.dtype().clone()) + .collect(), + ) } /// Create a new `StructArray` with the given length, but without any fields. @@ -328,8 +332,9 @@ impl StructData { .collect(); Self { - len: length, - dtype: DType::Struct(dtype, validity.nullability()), + names: dtype.names().clone(), + nullability: validity.nullability(), + fieldless_len: fields.is_empty().then_some(length), slots, } } @@ -401,10 +406,27 @@ impl StructData { } pub fn into_parts(self) -> StructArrayParts { - let validity = self.validity(); - let struct_fields = self.dtype.into_struct_fields(); - let fields: Arc<[ArrayRef]> = self - .slots + let StructData { + names, + slots, + nullability, + .. + } = self; + let validity = child_to_validity(&slots[VALIDITY_SLOT], nullability); + let struct_fields = StructFields::new( + names, + slots + .iter() + .skip(FIELDS_OFFSET) + .map(|slot| { + slot.as_ref() + .vortex_expect("StructArray field slot") + .dtype() + .clone() + }) + .collect(), + ); + let fields: Arc<[ArrayRef]> = slots .into_iter() .skip(FIELDS_OFFSET) .map(|s| s.vortex_expect("StructArray field slot")) @@ -491,6 +513,7 @@ impl StructData { let name = name.into(); let struct_dtype = self.struct_fields().clone(); + let len = self.len(); let position = struct_dtype .names() @@ -512,11 +535,40 @@ impl StructData { if let Ok(new_dtype) = struct_dtype.without_field(position) { self.slots = new_slots; - self.dtype = DType::Struct(new_dtype, self.dtype.nullability()); + self.names = new_dtype.names().clone(); + self.fieldless_len = self + .slots + .get(FIELDS_OFFSET) + .and_then(|slot| slot.as_ref()) + .map(|_| None) + .unwrap_or(Some(len)); return Some(field); } None } + + pub fn validate_against_outer(&self, dtype: &DType, len: usize) -> VortexResult<()> { + match dtype { + DType::Struct(_, _) => {} + _ => vortex_bail!("Expected struct dtype, found {:?}", dtype), + } + if self.len() != len { + vortex_bail!( + InvalidArgument: "StructArray length {} does not match outer length {}", + self.len(), + len + ); + } + let data_dtype = self.dtype(); + if &data_dtype != dtype { + vortex_bail!( + InvalidArgument: "StructArray dtype {} does not match outer dtype {}", + data_dtype, + dtype + ); + } + Ok(()) + } } impl Array { @@ -586,8 +638,8 @@ impl Array { Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) } - /// Decompose this struct array into its constituent parts. - pub fn into_parts(self) -> StructArrayParts { + /// Decompose this struct array into its encoding-specific constituent parts. + pub fn into_encoding_parts(self) -> StructArrayParts { self.into_data().into_parts() } @@ -641,6 +693,6 @@ impl StructData { .chain(once(array)) .collect(); - Self::try_new_with_dtype(children, new_fields, self.len, self.validity()) + Self::try_new_with_dtype(children, new_fields, self.len(), self.validity()) } } diff --git a/vortex-array/src/arrays/struct_/compute/rules.rs b/vortex-array/src/arrays/struct_/compute/rules.rs index ddb471e8930..4d21b190151 100644 --- a/vortex-array/src/arrays/struct_/compute/rules.rs +++ b/vortex-array/src/arrays/struct_/compute/rules.rs @@ -82,7 +82,7 @@ impl ArrayParentReduceRule for StructCastPushDownRule { } else { array .validity() - .into_non_nullable(array.len) + .into_non_nullable(array.len()) .ok_or_else(|| vortex_err!("Failed to cast nullable struct to non-nullable"))? }; diff --git a/vortex-array/src/arrays/struct_/compute/zip.rs b/vortex-array/src/arrays/struct_/compute/zip.rs index bf13f4f72cf..75fe4288dd6 100644 --- a/vortex-array/src/arrays/struct_/compute/zip.rs +++ b/vortex-array/src/arrays/struct_/compute/zip.rs @@ -54,7 +54,7 @@ impl ZipKernel for Struct { let combined = (v1m.bitand(&mask_mask)).bitor(&v2m.bitand(&mask_mask.not())); Validity::from_mask( combined, - if_true.dtype.nullability() | if_false.dtype.nullability(), + if_true.dtype().nullability() | if_false.dtype().nullability(), ) } }; diff --git a/vortex-array/src/arrays/struct_/vtable/mod.rs b/vortex-array/src/arrays/struct_/vtable/mod.rs index 6034fa8e2a0..33fa42f91b3 100644 --- a/vortex-array/src/arrays/struct_/vtable/mod.rs +++ b/vortex-array/src/arrays/struct_/vtable/mod.rs @@ -66,6 +66,10 @@ impl VTable for Struct { 0 } + fn validate(&self, data: &StructData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate_against_outer(dtype, len) + } + fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { vortex_panic!("StructArray buffer index {idx} out of bounds") } @@ -81,7 +85,8 @@ impl VTable for Struct { fn deserialize( &self, dtype: &DType, - len: usize, metadata: &[u8], + len: usize, + metadata: &[u8], _buffers: &[BufferHandle], children: &dyn ArrayChildren, diff --git a/vortex-array/src/arrays/varbin/array.rs b/vortex-array/src/arrays/varbin/array.rs index ec7a403dbb7..4fdb1bda6c4 100644 --- a/vortex-array/src/arrays/varbin/array.rs +++ b/vortex-array/src/arrays/varbin/array.rs @@ -5,6 +5,7 @@ use num_traits::AsPrimitive; use vortex_buffer::ByteBuffer; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_mask::Mask; @@ -33,12 +34,29 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["offsets", "validity"]; #[derive(Clone, Debug)] pub struct VarBinData { - pub(super) dtype: DType, + pub(super) is_utf8: bool, + pub(super) nullability: Nullability, pub(super) bytes: BufferHandle, pub(super) slots: Vec>, } impl VarBinData { + fn dtype_parts(dtype: &DType) -> VortexResult<(bool, Nullability)> { + match dtype { + DType::Utf8(nullability) => Ok((true, *nullability)), + DType::Binary(nullability) => Ok((false, *nullability)), + _ => vortex_bail!(MismatchedTypes: "utf8 or binary", dtype), + } + } + + fn make_dtype(is_utf8: bool, nullability: Nullability) -> DType { + if is_utf8 { + DType::Utf8(nullability) + } else { + DType::Binary(nullability) + } + } + /// Creates a new `VarBinArray`. /// /// # Panics @@ -165,9 +183,12 @@ impl VarBinData { let len = offsets.len().saturating_sub(1); let validity_slot = validity_to_child(&validity, len); + let (is_utf8, nullability) = + Self::dtype_parts(&dtype).vortex_expect("VarBinArray dtype must be utf8 or binary"); Self { - dtype, + is_utf8, + nullability, bytes, slots: vec![Some(offsets), validity_slot], } @@ -273,8 +294,8 @@ impl VarBinData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + Self::make_dtype(self.is_utf8, self.nullability) } /// Returns `true` if this array is empty. @@ -285,7 +306,7 @@ impl VarBinData { /// Returns the [`Validity`] of this array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) } /// Returns the validity as a [`Mask`]. @@ -472,7 +493,23 @@ impl VarBinData { let offsets = self.slots[OFFSETS_SLOT] .take() .vortex_expect("VarBinArray offsets slot"); - (self.dtype, self.bytes, offsets, validity) + (self.dtype(), self.bytes, offsets, validity) + } + + pub fn validate_against_outer(&self, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + self.len() == len, + "VarBinArray length {} does not match outer length {}", + self.len(), + len + ); + vortex_ensure!( + self.dtype() == *dtype, + "VarBinArray dtype {} does not match outer dtype {}", + self.dtype(), + dtype + ); + Ok(()) } } diff --git a/vortex-array/src/arrays/varbin/vtable/mod.rs b/vortex-array/src/arrays/varbin/vtable/mod.rs index da69c44412f..8e66b4cb12a 100644 --- a/vortex-array/src/arrays/varbin/vtable/mod.rs +++ b/vortex-array/src/arrays/varbin/vtable/mod.rs @@ -76,6 +76,10 @@ impl VTable for VarBin { 1 } + fn validate(&self, data: &VarBinData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate_against_outer(dtype, len) + } + fn buffer(array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { match idx { 0 => array.bytes_handle().clone(), @@ -91,17 +95,20 @@ impl VTable for VarBin { } fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some(ProstMetadata(VarBinMetadata { - offsets_ptype: PType::try_from(array.offsets().dtype()) - .vortex_expect("Must be a valid PType") as i32, - }) - .serialize())) + Ok(Some( + ProstMetadata(VarBinMetadata { + offsets_ptype: PType::try_from(array.offsets().dtype()) + .vortex_expect("Must be a valid PType") as i32, + }) + .serialize(), + )) } fn deserialize( &self, dtype: &DType, - len: usize, metadata: &[u8], + len: usize, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, diff --git a/vortex-array/src/arrays/varbinview/array.rs b/vortex-array/src/arrays/varbinview/array.rs index 61ac959f5e7..ab06e8a9229 100644 --- a/vortex-array/src/arrays/varbinview/array.rs +++ b/vortex-array/src/arrays/varbinview/array.rs @@ -96,7 +96,8 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; #[derive(Clone, Debug)] pub struct VarBinViewData { pub(super) slots: Vec>, - pub(super) dtype: DType, + pub(super) is_utf8: bool, + pub(super) nullability: Nullability, pub(super) buffers: Arc<[BufferHandle]>, pub(super) views: BufferHandle, } @@ -109,6 +110,22 @@ pub struct VarBinViewArrayParts { } impl VarBinViewData { + fn dtype_parts(dtype: &DType) -> VortexResult<(bool, Nullability)> { + match dtype { + DType::Utf8(nullability) => Ok((true, *nullability)), + DType::Binary(nullability) => Ok((false, *nullability)), + _ => vortex_bail!(InvalidArgument: "invalid DType {dtype} for `VarBinViewArray`"), + } + } + + fn make_dtype(is_utf8: bool, nullability: Nullability) -> DType { + if is_utf8 { + DType::Utf8(nullability) + } else { + DType::Binary(nullability) + } + } + /// Build the slots vector for this array. pub(super) fn make_slots(validity: &Validity, len: usize) -> Vec> { vec![validity_to_child(validity, len)] @@ -262,11 +279,14 @@ impl VarBinViewData { ) -> Self { let len = views.len() / size_of::(); let slots = Self::make_slots(&validity, len); + let (is_utf8, nullability) = + Self::dtype_parts(&dtype).vortex_expect("VarBinViewArray dtype must be utf8 or binary"); Self { slots, views, buffers, - dtype, + is_utf8, + nullability, } } @@ -365,8 +385,8 @@ impl VarBinViewData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + Self::make_dtype(self.is_utf8, self.nullability) } /// Returns `true` if this array is empty. @@ -377,7 +397,7 @@ impl VarBinViewData { /// Returns the [`Validity`] of this array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) } /// Returns the validity as a [`Mask`]. @@ -389,13 +409,29 @@ impl VarBinViewData { pub fn into_parts(self) -> VarBinViewArrayParts { let validity = self.validity(); VarBinViewArrayParts { - dtype: self.dtype, + dtype: self.dtype(), buffers: self.buffers, views: self.views, validity, } } + pub fn validate_against_outer(&self, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + self.len() == len, + "VarBinViewArray length {} does not match outer length {}", + self.len(), + len + ); + vortex_ensure!( + self.dtype() == *dtype, + "VarBinViewArray dtype {} does not match outer dtype {}", + self.dtype(), + dtype + ); + Ok(()) + } + /// Access to the primitive views buffer. /// /// Variable-sized binary view buffer contain a "view" child array, with 16-byte entries that diff --git a/vortex-array/src/arrays/varbinview/vtable/mod.rs b/vortex-array/src/arrays/varbinview/vtable/mod.rs index c1b997072ed..fa397991280 100644 --- a/vortex-array/src/arrays/varbinview/vtable/mod.rs +++ b/vortex-array/src/arrays/varbinview/vtable/mod.rs @@ -83,6 +83,10 @@ impl VTable for VarBinView { array.data_buffers().len() + 1 } + fn validate(&self, data: &VarBinViewData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate_against_outer(dtype, len) + } + fn buffer(array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { let ndata = array.data_buffers().len(); if idx < ndata { diff --git a/vortex-array/src/arrays/variant/mod.rs b/vortex-array/src/arrays/variant/mod.rs index 78d8d316f68..5ae1f503820 100644 --- a/vortex-array/src/arrays/variant/mod.rs +++ b/vortex-array/src/arrays/variant/mod.rs @@ -31,7 +31,9 @@ pub struct VariantData { impl VariantData { /// Creates a new VariantArray. Nullability comes from the child's dtype. pub fn new(child: ArrayRef) -> Self { - Self { slots: vec![Some(child)] } + Self { + slots: vec![Some(child)], + } } /// Returns the length of this array. diff --git a/vortex-array/src/arrays/variant/vtable/mod.rs b/vortex-array/src/arrays/variant/vtable/mod.rs index 80855420860..7b36379a76a 100644 --- a/vortex-array/src/arrays/variant/vtable/mod.rs +++ b/vortex-array/src/arrays/variant/vtable/mod.rs @@ -76,7 +76,8 @@ impl VTable for Variant { fn deserialize( &self, dtype: &DType, - len: usize, metadata: &[u8], + len: usize, + metadata: &[u8], _buffers: &[BufferHandle], children: &dyn ArrayChildren, diff --git a/vortex-array/src/arrow/executor/struct_.rs b/vortex-array/src/arrow/executor/struct_.rs index eab871de2eb..02deda5913d 100644 --- a/vortex-array/src/arrow/executor/struct_.rs +++ b/vortex-array/src/arrow/executor/struct_.rs @@ -55,7 +55,7 @@ pub(super) fn to_arrow_struct( fields, struct_fields, .. - } = array.into_parts(); + } = array.into_encoding_parts(); let validity = to_arrow_null_buffer(validity, len, ctx)?; return create_from_fields( target_fields.ok_or_else(|| struct_fields.names().clone()), @@ -103,7 +103,7 @@ pub(super) fn to_arrow_struct( fields, struct_fields, .. - } = struct_array.into_parts(); + } = struct_array.into_encoding_parts(); let validity = to_arrow_null_buffer(validity, len, ctx)?; create_from_fields( diff --git a/vortex-array/src/canonical.rs b/vortex-array/src/canonical.rs index e1220bf4837..c4714b07ed1 100644 --- a/vortex-array/src/canonical.rs +++ b/vortex-array/src/canonical.rs @@ -549,7 +549,7 @@ impl Executable for CanonicalValidity { offset, len, validity, - } = b.into_parts(); + } = b.into_encoding_parts(); Ok(CanonicalValidity(Canonical::Bool( BoolArray::try_new_from_handle(bits, offset, len, validity.execute(ctx)?)?, ))) @@ -624,7 +624,7 @@ impl Executable for CanonicalValidity { struct_fields, fields, validity, - } = st.into_parts(); + } = st.into_encoding_parts(); Ok(CanonicalValidity(Canonical::Struct(unsafe { StructArray::new_unchecked(fields, struct_fields, len, validity.execute(ctx)?) }))) @@ -669,7 +669,7 @@ impl Executable for RecursiveCanonical { offset, len, validity, - } = b.into_parts(); + } = b.into_encoding_parts(); Ok(RecursiveCanonical(Canonical::Bool( BoolArray::try_new_from_handle(bits, offset, len, validity.execute(ctx)?)?, ))) @@ -754,7 +754,7 @@ impl Executable for RecursiveCanonical { struct_fields, fields, validity, - } = st.into_parts(); + } = st.into_encoding_parts(); let executed_fields = fields .iter() .map(|f| Ok(f.clone().execute::(ctx)?.0.into_array())) diff --git a/vortex-array/src/display/extractors/metadata.rs b/vortex-array/src/display/extractors/metadata.rs index 20af3ead0a8..3bbe9c995e0 100644 --- a/vortex-array/src/display/extractors/metadata.rs +++ b/vortex-array/src/display/extractors/metadata.rs @@ -22,6 +22,7 @@ impl TreeExtractor for MetadataExtractor { let (indent, f) = f.parts(); write!(f, "{indent}metadata: ")?; match array.metadata() { + Ok(Some(metadata)) if metadata.is_empty() => write!(f, "EmptyMetadata")?, Ok(Some(metadata)) => Debug::fmt(&metadata, f)?, Ok(None) => write!(f, "")?, Err(err) => write!(f, "")?, diff --git a/vortex-array/src/serde.rs b/vortex-array/src/serde.rs index b3d89b3b2a0..6f296ac3885 100644 --- a/vortex-array/src/serde.rs +++ b/vortex-array/src/serde.rs @@ -487,7 +487,7 @@ impl SerializedArray { /// without needing to access the actual buffer data. pub fn buffer_lengths(&self) -> Vec { let fb_array = root::(self.flatbuffer.as_ref()) - .vortex_expect("ArrayParts flatbuffer must be a valid Array"); + .vortex_expect("SerializedArray flatbuffer must be a valid Array"); fb_array .buffers() .map(|buffers| buffers.iter().map(|b| b.length() as usize).collect()) @@ -525,11 +525,11 @@ impl SerializedArray { /// Create an [`SerializedArray`] from a raw array tree flatbuffer (metadata only). /// - /// This constructor creates an `ArrayParts` with no buffer data, useful for + /// This constructor creates a `SerializedArray` with no buffer data, useful for /// inspecting the metadata when the actual buffer data is not needed /// (e.g., displaying buffer sizes from inlined array tree metadata). /// - /// Note: Calling `buffer()` on the returned `ArrayParts` will fail since + /// Note: Calling `buffer()` on the returned `SerializedArray` will fail since /// no actual buffer data is available. pub fn from_array_tree(array_tree: impl Into) -> VortexResult { let (flatbuffer, flatbuffer_loc) = Self::validate_array_tree(array_tree)?; @@ -643,7 +643,7 @@ impl TryFrom for SerializedArray { fn try_from(value: ByteBuffer) -> Result { // The final 4 bytes contain the length of the flatbuffer. if value.len() < 4 { - vortex_bail!("ArrayParts buffer is too short"); + vortex_bail!("SerializedArray buffer is too short"); } // We align each buffer individually, so we remove alignment requirements on the buffer. @@ -651,7 +651,7 @@ impl TryFrom for SerializedArray { let fb_length = u32::try_from_le_bytes(&value.as_slice()[value.len() - 4..])? as usize; if value.len() < 4 + fb_length { - vortex_bail!("ArrayParts buffer is too short for flatbuffer"); + vortex_bail!("SerializedArray buffer is too short for flatbuffer"); } let fb_offset = value.len() - 4 - fb_length; diff --git a/vortex-btrblocks/src/schemes/integer.rs b/vortex-btrblocks/src/schemes/integer.rs index 1f359106185..70d86412194 100644 --- a/vortex-btrblocks/src/schemes/integer.rs +++ b/vortex-btrblocks/src/schemes/integer.rs @@ -340,10 +340,7 @@ impl Scheme for BitPackingScheme { let nullability = packed.dtype().nullability(); let mut packed_data = packed.into_data(); - let patches = packed_data - .patches(len) - .map(compress_patches) - .transpose()?; + let patches = packed_data.patches(len).map(compress_patches).transpose()?; packed_data.replace_patches(patches); let parts = packed_data.into_parts(len, nullability); diff --git a/vortex-ipc/public-api.lock b/vortex-ipc/public-api.lock index 101ac7940ef..2c79b3ef0cb 100644 --- a/vortex-ipc/public-api.lock +++ b/vortex-ipc/public-api.lock @@ -46,7 +46,7 @@ pub mod vortex_ipc::messages pub enum vortex_ipc::messages::DecoderMessage -pub vortex_ipc::messages::DecoderMessage::Array((vortex_array::serde::ArrayParts, vortex_session::registry::ReadContext, usize)) +pub vortex_ipc::messages::DecoderMessage::Array((vortex_array::serde::SerializedArray, vortex_session::registry::ReadContext, usize)) pub vortex_ipc::messages::DecoderMessage::Buffer(vortex_buffer::ByteBuffer) diff --git a/vortex-python/python/vortex/__init__.py b/vortex-python/python/vortex/__init__.py index 40c3e7c4a91..5396f50b7a0 100644 --- a/vortex-python/python/vortex/__init__.py +++ b/vortex-python/python/vortex/__init__.py @@ -74,7 +74,7 @@ Utf8Scalar, scalar, ) -from ._lib.serde import ArrayContext, ArrayParts # pyright: ignore[reportMissingModuleSource] +from ._lib.serde import ArrayContext, SerializedArray # pyright: ignore[reportMissingModuleSource] from .arrays import ( Array, PyArray, @@ -177,7 +177,7 @@ "ExtensionScalar", # Serde "ArrayContext", - "ArrayParts", + "SerializedArray", # Pickle "_unpickle_array", # File diff --git a/vortex-python/python/vortex/_lib/serde.pyi b/vortex-python/python/vortex/_lib/serde.pyi index 626946a7e6c..b4d052b3e47 100644 --- a/vortex-python/python/vortex/_lib/serde.pyi +++ b/vortex-python/python/vortex/_lib/serde.pyi @@ -10,9 +10,9 @@ from .arrays import Array from .dtype import DType @final -class ArrayParts: +class SerializedArray: @staticmethod - def parse(data: bytes) -> ArrayParts: ... + def parse(data: bytes) -> SerializedArray: ... @property def metadata(self) -> bytes | None: ... @property @@ -22,7 +22,7 @@ class ArrayParts: @property def nchildren(self) -> int: ... @property - def children(self) -> list[ArrayParts]: ... + def children(self) -> list[SerializedArray]: ... def decode(self, ctx: ArrayContext, dtype: DType, len: int) -> pa.Array[pa.Scalar[pa.DataType]]: ... @final diff --git a/vortex-python/python/vortex/arrays.py b/vortex-python/python/vortex/arrays.py index 4180510d501..5181400cb32 100644 --- a/vortex-python/python/vortex/arrays.py +++ b/vortex-python/python/vortex/arrays.py @@ -13,7 +13,7 @@ from vortex._lib.dtype import DType # pyright: ignore[reportMissingModuleSource] from vortex._lib.serde import ( # pyright: ignore[reportMissingModuleSource] ArrayContext, - ArrayParts, + SerializedArray, decode_ipc_array_buffers, ) @@ -463,12 +463,12 @@ def dtype(self) -> DType: @classmethod @abc.abstractmethod - def decode(cls, parts: ArrayParts, ctx: ArrayContext, dtype: DType, len: int) -> Array: + def decode(cls, parts: SerializedArray, ctx: ArrayContext, dtype: DType, len: int) -> Array: """Decode an array from its component parts. - :class:`ArrayParts` contains the metadata, buffers and child :class:`ArrayParts` that represent the - current array. Implementations of this function should validate this information, and then construct - a new array. + :class:`SerializedArray` contains the metadata, buffers and child :class:`SerializedArray` + that represent the current array. Implementations of this function should validate this + information, and then construct a new array. """ diff --git a/vortex-python/python/vortex/serde.py b/vortex-python/python/vortex/serde.py index cb1f7f6c9a0..c30ceef4058 100644 --- a/vortex-python/python/vortex/serde.py +++ b/vortex-python/python/vortex/serde.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -from vortex._lib.serde import ArrayContext, ArrayParts # pyright: ignore[reportMissingModuleSource] +from vortex._lib.serde import ArrayContext, SerializedArray # pyright: ignore[reportMissingModuleSource] -__all__ = ["ArrayParts", "ArrayContext"] +__all__ = ["SerializedArray", "ArrayContext"] diff --git a/vortex-python/src/serde/mod.rs b/vortex-python/src/serde/mod.rs index e3290821194..c4e4cc95948 100644 --- a/vortex-python/src/serde/mod.rs +++ b/vortex-python/src/serde/mod.rs @@ -21,7 +21,7 @@ use crate::error::PyVortexResult; use crate::install_module; use crate::serde::context::PyArrayContext; use crate::serde::context::PyReadContext; -use crate::serde::parts::PyArrayParts; +use crate::serde::parts::PySerializedArray; /// Register serde functions and classes. pub(crate) fn init(py: Python, parent: &Bound) -> PyResult<()> { @@ -29,7 +29,7 @@ pub(crate) fn init(py: Python, parent: &Bound) -> PyResult<()> { parent.add_submodule(&m)?; install_module("vortex._lib.serde", &m)?; - m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_function(wrap_pyfunction!(decode_ipc_array, &m)?)?; diff --git a/vortex-python/src/serde/parts.rs b/vortex-python/src/serde/parts.rs index 5f112268e71..e05a6b6d1ac 100644 --- a/vortex-python/src/serde/parts.rs +++ b/vortex-python/src/serde/parts.rs @@ -20,13 +20,13 @@ use crate::dtype::PyDType; use crate::error::PyVortexResult; use crate::serde::context::PyReadContext; -/// ArrayParts is a parsed representation of a serialized array. +/// SerializedArray is a parsed representation of a serialized array. /// /// It can be decoded into a full array using the `decode` method. -#[pyclass(name = "ArrayParts", module = "vortex", frozen)] -pub(crate) struct PyArrayParts(SerializedArray); +#[pyclass(name = "SerializedArray", module = "vortex", frozen)] +pub(crate) struct PySerializedArray(SerializedArray); -impl Deref for PyArrayParts { +impl Deref for PySerializedArray { type Target = SerializedArray; fn deref(&self) -> &Self::Target { @@ -34,20 +34,20 @@ impl Deref for PyArrayParts { } } -impl From for PyArrayParts { +impl From for PySerializedArray { fn from(parts: SerializedArray) -> Self { Self(parts) } } #[pymethods] -impl PyArrayParts { +impl PySerializedArray { /// Parse a serialized array into its parts. #[staticmethod] - fn parse(data: &[u8]) -> PyVortexResult { + fn parse(data: &[u8]) -> PyVortexResult { // TODO(ngates): create a buffer from a slice of bytes? let buffer = ByteBuffer::copy_from(data); - Ok(PyArrayParts(SerializedArray::try_from(buffer)?)) + Ok(PySerializedArray(SerializedArray::try_from(buffer)?)) } /// Decode the array parts into a full array. @@ -115,12 +115,12 @@ impl PyArrayParts { self.0.nchildren() } - /// Return the child :class:`~vortex.ArrayParts` of the array. + /// Return the child :class:`~vortex.SerializedArray` of the array. #[getter] - fn children(&self) -> Vec { + fn children(&self) -> Vec { (0..self.0.nchildren()) .map(|idx| self.0.child(idx)) - .map(PyArrayParts) + .map(PySerializedArray) .collect() } } diff --git a/vortex-python/test/test_pyarray.py b/vortex-python/test/test_pyarray.py index 5713ff669f0..f29ca41408f 100644 --- a/vortex-python/test/test_pyarray.py +++ b/vortex-python/test/test_pyarray.py @@ -91,7 +91,7 @@ def encode(cls, array: pa.Array[pa.Scalar[pa.DataType]], config: ChunkConfig | N @override @classmethod - def decode(cls, parts: vx.ArrayParts, ctx: vx.ArrayContext, dtype: vx.DType, len: int) -> vx.Array: + def decode(cls, parts: vx.SerializedArray, ctx: vx.ArrayContext, dtype: vx.DType, len: int) -> vx.Array: """Decode the serialized array parts into an array.""" assert pco raise NotImplementedError From 7ea2ef8ff0952034828d336f630cf012c8779821 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Fri, 3 Apr 2026 09:20:56 -0400 Subject: [PATCH 05/15] Advanced frame parsing Signed-off-by: Nicholas Gates --- encodings/alp/public-api.lock | 38 +- encodings/alp/src/alp_rd/array.rs | 8 +- encodings/datetime-parts/public-api.lock | 20 - encodings/datetime-parts/src/array.rs | 10 +- encodings/decimal-byte-parts/public-api.lock | 14 +- .../src/decimal_byte_parts/mod.rs | 14 +- encodings/fastlanes/public-api.lock | 32 +- .../fastlanes/src/bitpacking/array/mod.rs | 6 +- encodings/fastlanes/src/bitpacking/mod.rs | 2 +- .../fastlanes/src/bitpacking/vtable/mod.rs | 4 +- encodings/runend/public-api.lock | 20 +- encodings/runend/src/array.rs | 35 +- encodings/runend/src/arrow.rs | 38 +- encodings/sequence/public-api.lock | 22 +- encodings/sequence/src/array.rs | 46 +- encodings/sequence/src/lib.rs | 2 +- encodings/zstd/public-api.lock | 42 +- encodings/zstd/src/array.rs | 42 +- vortex-array/public-api.lock | 536 +++++++++++++----- vortex-array/src/array/erased.rs | 5 + vortex-array/src/array/mod.rs | 9 + vortex-array/src/array/typed.rs | 27 +- vortex-array/src/array/vtable/dyn_.rs | 21 +- vortex-array/src/array/vtable/mod.rs | 13 +- vortex-array/src/arrays/bool/array.rs | 11 +- vortex-array/src/arrays/bool/mod.rs | 2 +- vortex-array/src/arrays/decimal/array.rs | 6 +- vortex-array/src/arrays/decimal/mod.rs | 2 +- vortex-array/src/arrays/dict/array.rs | 8 +- vortex-array/src/arrays/dict/vtable/mod.rs | 4 +- vortex-array/src/arrays/filter/array.rs | 36 +- vortex-array/src/arrays/filter/mod.rs | 2 +- vortex-array/src/arrays/filter/rules.rs | 4 +- vortex-array/src/arrays/filter/vtable.rs | 22 + .../src/arrays/fixed_size_list/array.rs | 2 +- vortex-array/src/arrays/list/array.rs | 37 +- vortex-array/src/arrays/list/mod.rs | 2 +- vortex-array/src/arrays/listview/array.rs | 6 +- vortex-array/src/arrays/listview/mod.rs | 2 +- vortex-array/src/arrays/masked/array.rs | 2 +- .../src/arrays/patched/compute/compare.rs | 7 +- .../src/arrays/patched/compute/take.rs | 4 +- vortex-array/src/arrays/patched/vtable/mod.rs | 4 +- .../src/arrays/primitive/array/mod.rs | 6 +- vortex-array/src/arrays/primitive/mod.rs | 2 +- vortex-array/src/arrays/slice/array.rs | 25 +- vortex-array/src/arrays/slice/mod.rs | 2 +- vortex-array/src/arrays/slice/vtable.rs | 22 + vortex-array/src/arrays/struct_/array.rs | 15 +- vortex-array/src/arrays/struct_/mod.rs | 2 +- vortex-array/src/arrays/varbinview/array.rs | 6 +- vortex-array/src/arrays/varbinview/mod.rs | 2 +- vortex-array/src/arrays/variant/mod.rs | 19 +- vortex-array/src/arrays/variant/vtable/mod.rs | 29 +- vortex-array/src/arrow/executor/dictionary.rs | 4 +- vortex-array/src/arrow/executor/list.rs | 4 +- vortex-array/src/arrow/executor/list_view.rs | 4 +- vortex-array/src/arrow/executor/struct_.rs | 16 +- vortex-array/src/canonical.rs | 52 +- .../src/display/extractors/metadata.rs | 9 +- vortex-array/src/stats/array.rs | 4 + vortex-cuda/benches/zstd_cuda.rs | 4 +- vortex-cuda/src/arrow/canonical.rs | 22 +- vortex-cuda/src/arrow/varbinview.rs | 4 +- vortex-cuda/src/canonical.rs | 24 +- .../src/dynamic_dispatch/plan_builder.rs | 4 +- vortex-cuda/src/executor.rs | 6 +- vortex-cuda/src/kernel/arrays/dict.rs | 30 +- vortex-cuda/src/kernel/encodings/alp.rs | 8 +- vortex-cuda/src/kernel/encodings/bitpacked.rs | 6 +- .../src/kernel/encodings/date_time_parts.rs | 8 +- .../kernel/encodings/decimal_byte_parts.rs | 14 +- vortex-cuda/src/kernel/encodings/for_.rs | 8 +- vortex-cuda/src/kernel/encodings/runend.rs | 10 +- vortex-cuda/src/kernel/encodings/sequence.rs | 9 +- vortex-cuda/src/kernel/encodings/zigzag.rs | 4 +- vortex-cuda/src/kernel/encodings/zstd.rs | 8 +- vortex-cuda/src/kernel/filter/decimal.rs | 4 +- vortex-cuda/src/kernel/filter/mod.rs | 4 +- vortex-cuda/src/kernel/filter/primitive.rs | 4 +- vortex-cuda/src/kernel/filter/varbinview.rs | 4 +- vortex-cuda/src/kernel/patches/mod.rs | 10 +- vortex-cuda/src/kernel/slice/mod.rs | 4 +- vortex-duckdb/src/exporter/decimal.rs | 4 +- vortex-duckdb/src/exporter/list.rs | 4 +- vortex-duckdb/src/exporter/list_view.rs | 4 +- vortex-duckdb/src/exporter/run_end.rs | 4 +- vortex-duckdb/src/exporter/struct_.rs | 6 +- vortex-duckdb/src/exporter/varbinview.rs | 4 +- vortex-python/src/arrays/py/array.rs | 6 +- 90 files changed, 958 insertions(+), 665 deletions(-) diff --git a/encodings/alp/public-api.lock b/encodings/alp/public-api.lock index a130df6493b..6c0385166fb 100644 --- a/encodings/alp/public-api.lock +++ b/encodings/alp/public-api.lock @@ -232,29 +232,11 @@ impl vortex_array::scalar_fn::fns::mask::kernel::MaskReduce for vortex_alp::ALPR pub fn vortex_alp::ALPRD::mask(array: vortex_array::array::view::ArrayView<'_, Self>, mask: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_alp::ALPRDArrayParts - -pub vortex_alp::ALPRDArrayParts::left_parts: vortex_array::array::erased::ArrayRef - -pub vortex_alp::ALPRDArrayParts::left_parts_dictionary: vortex_buffer::buffer::Buffer - -pub vortex_alp::ALPRDArrayParts::left_parts_patches: core::option::Option - -pub vortex_alp::ALPRDArrayParts::right_parts: vortex_array::array::erased::ArrayRef - -impl core::clone::Clone for vortex_alp::ALPRDArrayParts - -pub fn vortex_alp::ALPRDArrayParts::clone(&self) -> vortex_alp::ALPRDArrayParts - -impl core::fmt::Debug for vortex_alp::ALPRDArrayParts - -pub fn vortex_alp::ALPRDArrayParts::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub struct vortex_alp::ALPRDData impl vortex_alp::ALPRDData -pub fn vortex_alp::ALPRDData::into_parts(self) -> vortex_alp::ALPRDArrayParts +pub fn vortex_alp::ALPRDData::into_parts(self) -> vortex_alp::ALPRDDataParts pub fn vortex_alp::ALPRDData::left_parts(&self) -> &vortex_array::array::erased::ArrayRef @@ -278,6 +260,24 @@ impl core::fmt::Debug for vortex_alp::ALPRDData pub fn vortex_alp::ALPRDData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct vortex_alp::ALPRDDataParts + +pub vortex_alp::ALPRDDataParts::left_parts: vortex_array::array::erased::ArrayRef + +pub vortex_alp::ALPRDDataParts::left_parts_dictionary: vortex_buffer::buffer::Buffer + +pub vortex_alp::ALPRDDataParts::left_parts_patches: core::option::Option + +pub vortex_alp::ALPRDDataParts::right_parts: vortex_array::array::erased::ArrayRef + +impl core::clone::Clone for vortex_alp::ALPRDDataParts + +pub fn vortex_alp::ALPRDDataParts::clone(&self) -> vortex_alp::ALPRDDataParts + +impl core::fmt::Debug for vortex_alp::ALPRDDataParts + +pub fn vortex_alp::ALPRDDataParts::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub struct vortex_alp::ALPRDMetadata impl vortex_alp::ALPRDMetadata diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index a785fc28552..8c1cd630ce6 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -252,7 +252,7 @@ impl VTable for ALPRD { let dtype = array.dtype().clone(); let right_bit_width = array.right_bit_width(); - let ALPRDArrayParts { + let ALPRDDataParts { left_parts, right_parts, left_parts_dictionary, @@ -348,7 +348,7 @@ pub struct ALPRDData { } #[derive(Clone, Debug)] -pub struct ALPRDArrayParts { +pub struct ALPRDDataParts { pub left_parts: ArrayRef, pub left_parts_patches: Option, pub left_parts_dictionary: Buffer, @@ -578,14 +578,14 @@ impl ALPRDData { } /// Return all the owned parts of the array - pub fn into_parts(mut self) -> ALPRDArrayParts { + pub fn into_parts(mut self) -> ALPRDDataParts { let left_parts = self.slots[LEFT_PARTS_SLOT] .take() .vortex_expect("ALPRDArray left_parts slot"); let right_parts = self.slots[RIGHT_PARTS_SLOT] .take() .vortex_expect("ALPRDArray right_parts slot"); - ALPRDArrayParts { + ALPRDDataParts { left_parts, left_parts_patches: self.left_parts_patches, left_parts_dictionary: self.left_parts_dictionary, diff --git a/encodings/datetime-parts/public-api.lock b/encodings/datetime-parts/public-api.lock index d10034f6527..1daadb646fe 100644 --- a/encodings/datetime-parts/public-api.lock +++ b/encodings/datetime-parts/public-api.lock @@ -88,24 +88,6 @@ impl vortex_array::scalar_fn::fns::mask::kernel::MaskReduce for vortex_datetime_ pub fn vortex_datetime_parts::DateTimeParts::mask(array: vortex_array::array::view::ArrayView<'_, Self>, mask: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_datetime_parts::DateTimePartsArrayParts - -pub vortex_datetime_parts::DateTimePartsArrayParts::days: vortex_array::array::erased::ArrayRef - -pub vortex_datetime_parts::DateTimePartsArrayParts::dtype: vortex_array::dtype::DType - -pub vortex_datetime_parts::DateTimePartsArrayParts::seconds: vortex_array::array::erased::ArrayRef - -pub vortex_datetime_parts::DateTimePartsArrayParts::subseconds: vortex_array::array::erased::ArrayRef - -impl core::clone::Clone for vortex_datetime_parts::DateTimePartsArrayParts - -pub fn vortex_datetime_parts::DateTimePartsArrayParts::clone(&self) -> vortex_datetime_parts::DateTimePartsArrayParts - -impl core::fmt::Debug for vortex_datetime_parts::DateTimePartsArrayParts - -pub fn vortex_datetime_parts::DateTimePartsArrayParts::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub struct vortex_datetime_parts::DateTimePartsData impl vortex_datetime_parts::DateTimePartsData @@ -120,8 +102,6 @@ pub fn vortex_datetime_parts::DateTimePartsData::seconds(&self) -> &vortex_array pub fn vortex_datetime_parts::DateTimePartsData::subseconds(&self) -> &vortex_array::array::erased::ArrayRef -pub fn vortex_datetime_parts::DateTimePartsData::try_new(dtype: vortex_array::dtype::DType, days: vortex_array::array::erased::ArrayRef, seconds: vortex_array::array::erased::ArrayRef, subseconds: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult - pub fn vortex_datetime_parts::DateTimePartsData::validate(dtype: &vortex_array::dtype::DType, days: &vortex_array::array::erased::ArrayRef, seconds: &vortex_array::array::erased::ArrayRef, subseconds: &vortex_array::array::erased::ArrayRef, len: usize) -> vortex_error::VortexResult<()> impl core::clone::Clone for vortex_datetime_parts::DateTimePartsData diff --git a/encodings/datetime-parts/src/array.rs b/encodings/datetime-parts/src/array.rs index 2214f1c505f..e4163ab85a8 100644 --- a/encodings/datetime-parts/src/array.rs +++ b/encodings/datetime-parts/src/array.rs @@ -221,14 +221,6 @@ pub struct DateTimePartsData { pub(super) slots: Vec>, } -#[derive(Clone, Debug)] -pub struct DateTimePartsArrayParts { - pub dtype: DType, - pub days: ArrayRef, - pub seconds: ArrayRef, - pub subseconds: ArrayRef, -} - #[derive(Clone, Debug)] pub struct DateTimeParts; @@ -292,7 +284,7 @@ impl DateTimePartsData { Ok(()) } - pub fn try_new( + pub(crate) fn try_new( dtype: DType, days: ArrayRef, seconds: ArrayRef, diff --git a/encodings/decimal-byte-parts/public-api.lock b/encodings/decimal-byte-parts/public-api.lock index 38ad5da9570..91d35435cab 100644 --- a/encodings/decimal-byte-parts/public-api.lock +++ b/encodings/decimal-byte-parts/public-api.lock @@ -86,22 +86,16 @@ impl vortex_array::scalar_fn::fns::mask::kernel::MaskReduce for vortex_decimal_b pub fn vortex_decimal_byte_parts::DecimalByteParts::mask(array: vortex_array::array::view::ArrayView<'_, Self>, mask: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_decimal_byte_parts::DecimalBytePartsArrayParts - -pub vortex_decimal_byte_parts::DecimalBytePartsArrayParts::dtype: vortex_array::dtype::DType - -pub vortex_decimal_byte_parts::DecimalBytePartsArrayParts::msp: vortex_array::array::erased::ArrayRef - pub struct vortex_decimal_byte_parts::DecimalBytePartsData impl vortex_decimal_byte_parts::DecimalBytePartsData +pub fn vortex_decimal_byte_parts::DecimalBytePartsData::into_parts(self) -> vortex_decimal_byte_parts::DecimalBytePartsDataParts + pub fn vortex_decimal_byte_parts::DecimalBytePartsData::is_empty(&self) -> bool pub fn vortex_decimal_byte_parts::DecimalBytePartsData::len(&self) -> usize -pub fn vortex_decimal_byte_parts::DecimalBytePartsData::try_new(msp: vortex_array::array::erased::ArrayRef, decimal_dtype: vortex_array::dtype::decimal::DecimalDType) -> vortex_error::VortexResult - pub fn vortex_decimal_byte_parts::DecimalBytePartsData::validate(msp: &vortex_array::array::erased::ArrayRef, decimal_dtype: vortex_array::dtype::decimal::DecimalDType, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> impl core::clone::Clone for vortex_decimal_byte_parts::DecimalBytePartsData @@ -112,6 +106,10 @@ impl core::fmt::Debug for vortex_decimal_byte_parts::DecimalBytePartsData pub fn vortex_decimal_byte_parts::DecimalBytePartsData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct vortex_decimal_byte_parts::DecimalBytePartsDataParts + +pub vortex_decimal_byte_parts::DecimalBytePartsDataParts::msp: vortex_array::array::erased::ArrayRef + pub struct vortex_decimal_byte_parts::DecimalBytesPartsMetadata impl vortex_decimal_byte_parts::DecimalBytesPartsMetadata diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs index d6fabe60737..a272f4d7f1b 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs @@ -193,13 +193,11 @@ pub struct DecimalBytePartsData { // NOTE: the lower_parts is currently unused, we reserve this field so that it is properly // read/written during serde, but provide no constructor to initialize this to anything // other than the empty Vec. - // Must update `DecimalBytePartsArrayParts` too. _lower_parts: Vec, } -pub struct DecimalBytePartsArrayParts { +pub struct DecimalBytePartsDataParts { pub msp: ArrayRef, - pub dtype: DType, } impl DecimalBytePartsData { @@ -222,7 +220,7 @@ impl DecimalBytePartsData { Ok(()) } - pub fn try_new(msp: ArrayRef, decimal_dtype: DecimalDType) -> VortexResult { + pub(crate) fn try_new(msp: ArrayRef, decimal_dtype: DecimalDType) -> VortexResult { let dtype = DType::Decimal(decimal_dtype, msp.dtype().nullability()); Self::validate(&msp, decimal_dtype, &dtype, msp.len())?; Ok(Self { @@ -246,6 +244,14 @@ impl DecimalBytePartsData { .as_ref() .vortex_expect("DecimalBytePartsArray msp slot") } + + pub fn into_parts(mut self) -> DecimalBytePartsDataParts { + DecimalBytePartsDataParts { + msp: self.slots[MSP_SLOT] + .take() + .vortex_expect("DecimalBytePartsArray msp slot"), + } + } } #[derive(Clone, Debug)] diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index fa62ac1ec02..ced1130a8f1 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -124,7 +124,7 @@ pub const vortex_fastlanes::BitPacked::ID: vortex_array::array::ArrayId pub fn vortex_fastlanes::BitPacked::encode(array: &vortex_array::array::erased::ArrayRef, bit_width: u8) -> vortex_error::VortexResult -pub fn vortex_fastlanes::BitPacked::into_parts(array: vortex_fastlanes::BitPackedArray) -> vortex_fastlanes::BitPackedArrayParts +pub fn vortex_fastlanes::BitPacked::into_parts(array: vortex_fastlanes::BitPackedArray) -> vortex_fastlanes::BitPackedDataParts pub fn vortex_fastlanes::BitPacked::try_new(packed: vortex_array::buffer::BufferHandle, ptype: vortex_array::dtype::ptype::PType, validity: vortex_array::validity::Validity, patches: core::option::Option, bit_width: u8, len: usize, offset: u16) -> vortex_error::VortexResult @@ -200,20 +200,6 @@ impl vortex_array::scalar_fn::fns::cast::kernel::CastReduce for vortex_fastlanes pub fn vortex_fastlanes::BitPacked::cast(array: vortex_array::array::view::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> -pub struct vortex_fastlanes::BitPackedArrayParts - -pub vortex_fastlanes::BitPackedArrayParts::bit_width: u8 - -pub vortex_fastlanes::BitPackedArrayParts::len: usize - -pub vortex_fastlanes::BitPackedArrayParts::offset: u16 - -pub vortex_fastlanes::BitPackedArrayParts::packed: vortex_array::buffer::BufferHandle - -pub vortex_fastlanes::BitPackedArrayParts::patches: core::option::Option - -pub vortex_fastlanes::BitPackedArrayParts::validity: vortex_array::validity::Validity - pub struct vortex_fastlanes::BitPackedData impl vortex_fastlanes::BitPackedData @@ -222,7 +208,7 @@ pub fn vortex_fastlanes::BitPackedData::bit_width(&self) -> u8 pub fn vortex_fastlanes::BitPackedData::encode(array: &vortex_array::array::erased::ArrayRef, bit_width: u8) -> vortex_error::VortexResult -pub fn vortex_fastlanes::BitPackedData::into_parts(self, len: usize, nullability: vortex_array::dtype::nullability::Nullability) -> vortex_fastlanes::BitPackedArrayParts +pub fn vortex_fastlanes::BitPackedData::into_parts(self, len: usize, nullability: vortex_array::dtype::nullability::Nullability) -> vortex_fastlanes::BitPackedDataParts pub fn vortex_fastlanes::BitPackedData::max_packed_value(&self) -> usize @@ -254,6 +240,20 @@ impl core::fmt::Debug for vortex_fastlanes::BitPackedData pub fn vortex_fastlanes::BitPackedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct vortex_fastlanes::BitPackedDataParts + +pub vortex_fastlanes::BitPackedDataParts::bit_width: u8 + +pub vortex_fastlanes::BitPackedDataParts::len: usize + +pub vortex_fastlanes::BitPackedDataParts::offset: u16 + +pub vortex_fastlanes::BitPackedDataParts::packed: vortex_array::buffer::BufferHandle + +pub vortex_fastlanes::BitPackedDataParts::patches: core::option::Option + +pub vortex_fastlanes::BitPackedDataParts::validity: vortex_array::validity::Validity + pub struct vortex_fastlanes::Delta impl vortex_fastlanes::Delta diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index 26ec86da064..0977bb8b34e 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -44,7 +44,7 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = [ "validity", ]; -pub struct BitPackedArrayParts { +pub struct BitPackedDataParts { pub offset: u16, pub bit_width: u8, pub len: usize, @@ -379,10 +379,10 @@ impl BitPackedData { (1 << self.bit_width()) - 1 } - pub fn into_parts(self, len: usize, nullability: Nullability) -> BitPackedArrayParts { + pub fn into_parts(self, len: usize, nullability: Nullability) -> BitPackedDataParts { let patches = self.patches(len); let validity = self.validity(nullability); - BitPackedArrayParts { + BitPackedDataParts { offset: self.offset, bit_width: self.bit_width, len, diff --git a/encodings/fastlanes/src/bitpacking/mod.rs b/encodings/fastlanes/src/bitpacking/mod.rs index 5a3a6ed0200..0e6e36804ea 100644 --- a/encodings/fastlanes/src/bitpacking/mod.rs +++ b/encodings/fastlanes/src/bitpacking/mod.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::BitPackedArrayParts; +pub use array::BitPackedDataParts; pub use array::BitPackedData; pub use array::bitpack_compress; pub use array::bitpack_decompress; diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs index 38e9239732b..ecebaa01183 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs @@ -37,7 +37,7 @@ use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; -use crate::BitPackedArrayParts; +use crate::BitPackedDataParts; use crate::BitPackedData; use crate::bitpack_decompress::unpack_array; use crate::bitpack_decompress::unpack_into_primitive_builder; @@ -331,7 +331,7 @@ impl BitPacked { Array::try_from_parts(ArrayParts::new(BitPacked, dtype, len, data)) } - pub fn into_parts(array: BitPackedArray) -> BitPackedArrayParts { + pub fn into_parts(array: BitPackedArray) -> BitPackedDataParts { let len = array.len(); let nullability = array.dtype().nullability(); array.into_data().into_parts(len, nullability) diff --git a/encodings/runend/public-api.lock b/encodings/runend/public-api.lock index 5822f213be4..528e3916681 100644 --- a/encodings/runend/public-api.lock +++ b/encodings/runend/public-api.lock @@ -106,12 +106,6 @@ impl vortex_array::scalar_fn::fns::fill_null::kernel::FillNullReduce for vortex_ pub fn vortex_runend::RunEnd::fill_null(array: vortex_array::array::view::ArrayView<'_, Self>, fill_value: &vortex_array::scalar::Scalar) -> vortex_error::VortexResult> -pub struct vortex_runend::RunEndArrayParts - -pub vortex_runend::RunEndArrayParts::ends: vortex_array::array::erased::ArrayRef - -pub vortex_runend::RunEndArrayParts::values: vortex_array::array::erased::ArrayRef - pub struct vortex_runend::RunEndData impl vortex_runend::RunEndData @@ -124,7 +118,7 @@ pub fn vortex_runend::RunEndData::ends(&self) -> &vortex_array::array::erased::A pub fn vortex_runend::RunEndData::find_physical_index(&self, index: usize) -> vortex_error::VortexResult -pub fn vortex_runend::RunEndData::into_parts(self) -> vortex_runend::RunEndArrayParts +pub fn vortex_runend::RunEndData::into_parts(self) -> vortex_runend::RunEndDataParts pub fn vortex_runend::RunEndData::new(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef) -> Self @@ -132,10 +126,6 @@ pub unsafe fn vortex_runend::RunEndData::new_unchecked(ends: vortex_array::array pub fn vortex_runend::RunEndData::offset(&self) -> usize -pub fn vortex_runend::RunEndData::try_new(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult - -pub fn vortex_runend::RunEndData::try_new_offset_length(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> vortex_error::VortexResult - pub fn vortex_runend::RunEndData::values(&self) -> &vortex_array::array::erased::ArrayRef impl core::clone::Clone for vortex_runend::RunEndData @@ -150,6 +140,14 @@ impl vortex_array::arrow::FromArrowArray pub fn vortex_runend::RunEndData::from_arrow(array: &arrow_array::array::run_array::RunArray, nullable: bool) -> vortex_error::VortexResult +pub struct vortex_runend::RunEndDataParts + +pub vortex_runend::RunEndDataParts::ends: vortex_array::array::erased::ArrayRef + +pub vortex_runend::RunEndDataParts::offset: usize + +pub vortex_runend::RunEndDataParts::values: vortex_array::array::erased::ArrayRef + pub struct vortex_runend::RunEndMetadata pub vortex_runend::RunEndMetadata::ends_ptype: i32 diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index bdeef325baa..8271e34d349 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -193,9 +193,10 @@ pub struct RunEndData { offset: usize, } -pub struct RunEndArrayParts { +pub struct RunEndDataParts { pub ends: ArrayRef, pub values: ArrayRef, + pub offset: usize, } #[derive(Clone, Debug)] @@ -291,12 +292,8 @@ impl RunEndData { return Ok(()); } - // Avoid building a non-empty array with zero logical length. + // Zero-length logical slices may retain run metadata from the source array. if length == 0 { - vortex_ensure!( - ends.is_empty(), - "run ends must be empty when length is zero" - ); return Ok(()); } @@ -323,8 +320,10 @@ impl RunEndData { // Validate the offset and length are valid for the given ends and values if offset != 0 && length != 0 { let first_run_end = usize::try_from(&ends.scalar_at(0)?)?; - if first_run_end <= offset { - vortex_bail!("First run end {first_run_end} must be bigger than offset {offset}"); + if first_run_end < offset { + vortex_bail!( + "First run end {first_run_end} must be >= offset {offset}" + ); } } @@ -341,7 +340,7 @@ impl RunEndData { impl RunEndData { /// Build a new `RunEndArray` from an array of run `ends` and an array of `values`. /// - /// Panics if any of the validation conditions described in [`RunEndData::try_new`] is + /// Panics if any of the validation conditions described in [`RunEnd::try_new`] is /// not satisfied. /// /// # Examples @@ -373,15 +372,15 @@ impl RunEndData { /// # Validation /// /// The `ends` must be non-nullable unsigned integers. - pub fn try_new(ends: ArrayRef, values: ArrayRef) -> VortexResult { + pub(crate) fn try_new(ends: ArrayRef, values: ArrayRef) -> VortexResult { let length = Self::logical_len_from_ends(&ends)?; Self::try_new_offset_length(ends, values, 0, length) } /// Construct a new sliced `RunEndArray` with the provided offset and length. /// - /// This performs all the same validation as [`RunEndData::try_new`]. - pub fn try_new_offset_length( + /// This performs all the same validation as [`RunEnd::try_new_offset_length`]. + pub(crate) fn try_new_offset_length( ends: ArrayRef, values: ArrayRef, offset: usize, @@ -399,10 +398,11 @@ impl RunEndData { /// /// # Safety /// - /// The caller must ensure that all the validation performed in [`RunEndData::try_new`] is + /// The caller must ensure that all the validation performed in + /// [`RunEnd::try_new_offset_length`] is /// satisfied before calling this function. /// - /// See [`RunEndData::try_new`] for the preconditions needed to build a new array. + /// See [`RunEnd::try_new_offset_length`] for the preconditions needed to build a new array. pub unsafe fn new_unchecked( ends: ArrayRef, values: ArrayRef, @@ -481,16 +481,15 @@ impl RunEndData { .vortex_expect("RunEndArray values slot") } - /// Split an `RunEndArray` into parts. - #[inline] - pub fn into_parts(mut self) -> RunEndArrayParts { - RunEndArrayParts { + pub fn into_parts(mut self) -> RunEndDataParts { + RunEndDataParts { ends: self.slots[ENDS_SLOT] .take() .vortex_expect("RunEndArray ends slot"), values: self.slots[VALUES_SLOT] .take() .vortex_expect("RunEndArray values slot"), + offset: self.offset, } } } diff --git a/encodings/runend/src/arrow.rs b/encodings/runend/src/arrow.rs index fd969c46c1c..e4fbd26f5f6 100644 --- a/encodings/runend/src/arrow.rs +++ b/encodings/runend/src/arrow.rs @@ -51,7 +51,7 @@ where ) }; - // SAFETY: arrow-rs enforces the RunEndArray invariants, we inherit their guarantees + // SAFETY: arrow-rs enforces the RunEndArray invariants, we inherit their guarantees. Ok(unsafe { RunEndData::new_unchecked(ends_slice, values_slice, offset, len) }) } } @@ -61,12 +61,14 @@ mod tests { use std::sync::Arc; use std::sync::LazyLock; + use arrow_array::Array as _; use arrow_array::Float64Array; use arrow_array::Int32Array; use arrow_array::Int64Array; use arrow_array::RunArray; use arrow_array::types::Int32Type; use arrow_array::types::Int64Type; + use arrow_array::types::RunEndIndexType; use arrow_schema::DataType; use arrow_schema::Field; use rstest::rstest; @@ -77,6 +79,7 @@ mod tests { use vortex_array::arrow::FromArrowArray; use vortex_array::assert_arrays_eq; use vortex_array::dtype::DType; + use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::session::ArraySession; @@ -84,11 +87,28 @@ mod tests { use vortex_error::VortexResult; use vortex_session::VortexSession; + use crate::RunEnd; use crate::RunEndData; static SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); + fn decode_run_array( + array: &RunArray, + nullable: bool, + ) -> VortexResult + where + R::Native: NativePType, + { + let data = RunEndData::from_arrow(array, nullable)?; + RunEnd::try_new_offset_length( + data.ends().clone(), + data.values().clone(), + data.offset(), + array.len(), + ) + } + #[test] fn test_arrow_run_array_to_vortex() -> VortexResult<()> { // Create an Arrow RunArray with UInt32 run ends and Int32 values @@ -99,7 +119,7 @@ mod tests { let arrow_run_array = RunArray::::try_new(&run_ends, &values).unwrap(); // Convert to Vortex - let vortex_array = RunEndData::from_arrow(&arrow_run_array, false)?; + let vortex_array = decode_run_array(&arrow_run_array, false)?; assert_arrays_eq!( vortex_array.into_array(), @@ -116,7 +136,7 @@ mod tests { let arrow_run_array = RunArray::::try_new(&run_ends, &values).unwrap(); // Convert to Vortex with nullable=true - let vortex_array = RunEndData::from_arrow(&arrow_run_array, true)?; + let vortex_array = decode_run_array(&arrow_run_array, true)?; assert_arrays_eq!( vortex_array.into_array(), @@ -140,7 +160,7 @@ mod tests { let arrow_run_array = RunArray::::try_new(&run_ends, &values).unwrap(); // Convert to Vortex - let vortex_array = RunEndData::from_arrow(&arrow_run_array, false)?; + let vortex_array = decode_run_array(&arrow_run_array, false)?; assert_arrays_eq!(vortex_array, buffer![1.5f64, 2.5, 2.5, 3.5].into_array()); Ok(()) @@ -160,7 +180,7 @@ mod tests { let sliced_array = arrow_run_array.slice(1, 6); // Convert the sliced array to Vortex - let vortex_array = RunEndData::from_arrow(&sliced_array, false)?; + let vortex_array = decode_run_array(&sliced_array, false)?; assert_arrays_eq!( vortex_array, buffer![100, 200, 200, 200, 300, 300].into_array() @@ -183,7 +203,7 @@ mod tests { let sliced_array = arrow_run_array.slice(4, 6); // Convert to Vortex with nullable=true - let vortex_array = RunEndData::from_arrow(&sliced_array, true)?; + let vortex_array = decode_run_array(&sliced_array, true)?; assert_arrays_eq!( vortex_array, @@ -214,7 +234,7 @@ mod tests { let sliced_array = arrow_run_array.slice(4, 0); // Convert to Vortex with nullable=true - let vortex_array = RunEndData::from_arrow(&sliced_array, true)?; + let vortex_array = decode_run_array(&sliced_array, true)?; // Verify properties assert_eq!(vortex_array.len(), 0); @@ -245,7 +265,7 @@ mod tests { &Int32Array::from(vec![3i32, 5, 8]), &Int32Array::from(vec![10, 20, 30]), )?; - let vortex_array = RunEndData::from_arrow(&original, false)?; + let vortex_array = decode_run_array(&original, false)?; let target = ree_type(DataType::Int32, DataType::Int32); let result = execute(vortex_array.into_array(), &target)?; @@ -279,7 +299,7 @@ mod tests { #[case] expected_values: &[i32], ) -> VortexResult<()> { let array = - RunEndData::encode(PrimitiveArray::from_iter(input.iter().copied()).into_array())?; + RunEnd::encode(PrimitiveArray::from_iter(input.iter().copied()).into_array())?; let sliced = array.into_array().slice(slice_range.clone())?; let target = ree_type(DataType::Int32, DataType::Int32); let result = execute(sliced, &target)?; diff --git a/encodings/sequence/public-api.lock b/encodings/sequence/public-api.lock index c511a00a026..f9b2995ced4 100644 --- a/encodings/sequence/public-api.lock +++ b/encodings/sequence/public-api.lock @@ -88,30 +88,18 @@ impl vortex_array::scalar_fn::fns::list_contains::kernel::ListContainsElementRed pub fn vortex_sequence::Sequence::list_contains(list: &vortex_array::array::erased::ArrayRef, element: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult> -pub struct vortex_sequence::SequenceArrayParts - -pub vortex_sequence::SequenceArrayParts::base: vortex_array::scalar::typed_view::primitive::pvalue::PValue - -pub vortex_sequence::SequenceArrayParts::multiplier: vortex_array::scalar::typed_view::primitive::pvalue::PValue - -pub vortex_sequence::SequenceArrayParts::ptype: vortex_array::dtype::ptype::PType - pub struct vortex_sequence::SequenceData impl vortex_sequence::SequenceData pub fn vortex_sequence::SequenceData::base(&self) -> vortex_array::scalar::typed_view::primitive::pvalue::PValue -pub fn vortex_sequence::SequenceData::into_parts(self) -> vortex_sequence::SequenceArrayParts +pub fn vortex_sequence::SequenceData::into_parts(self) -> vortex_sequence::SequenceDataParts pub fn vortex_sequence::SequenceData::multiplier(&self) -> vortex_array::scalar::typed_view::primitive::pvalue::PValue pub fn vortex_sequence::SequenceData::ptype(&self) -> vortex_array::dtype::ptype::PType -pub fn vortex_sequence::SequenceData::try_new(base: vortex_array::scalar::typed_view::primitive::pvalue::PValue, multiplier: vortex_array::scalar::typed_view::primitive::pvalue::PValue, ptype: vortex_array::dtype::ptype::PType, nullability: vortex_array::dtype::nullability::Nullability, length: usize) -> vortex_error::VortexResult - -pub fn vortex_sequence::SequenceData::try_new_typed>(base: T, multiplier: T, nullability: vortex_array::dtype::nullability::Nullability, length: usize) -> vortex_error::VortexResult - pub fn vortex_sequence::SequenceData::validate(base: vortex_array::scalar::typed_view::primitive::pvalue::PValue, multiplier: vortex_array::scalar::typed_view::primitive::pvalue::PValue, dtype: &vortex_array::dtype::DType, length: usize) -> vortex_error::VortexResult<()> impl core::clone::Clone for vortex_sequence::SequenceData @@ -122,6 +110,14 @@ impl core::fmt::Debug for vortex_sequence::SequenceData pub fn vortex_sequence::SequenceData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct vortex_sequence::SequenceDataParts + +pub vortex_sequence::SequenceDataParts::base: vortex_array::scalar::typed_view::primitive::pvalue::PValue + +pub vortex_sequence::SequenceDataParts::multiplier: vortex_array::scalar::typed_view::primitive::pvalue::PValue + +pub vortex_sequence::SequenceDataParts::ptype: vortex_array::dtype::ptype::PType + pub fn vortex_sequence::initialize(session: &mut vortex_session::VortexSession) pub fn vortex_sequence::sequence_encode(primitive_array: &vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult> diff --git a/encodings/sequence/src/array.rs b/encodings/sequence/src/array.rs index 6839c0a64c3..d8458499551 100644 --- a/encodings/sequence/src/array.rs +++ b/encodings/sequence/src/array.rs @@ -30,7 +30,6 @@ use vortex_array::scalar::PValue; use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::stats::StatsSet; use vortex_array::validity::Validity; use vortex_array::vtable; @@ -59,13 +58,6 @@ pub struct ProstSequenceMetadata { multiplier: Option, } -/// Components of [`SequenceArray`]. -pub struct SequenceArrayParts { - pub base: PValue, - pub multiplier: PValue, - pub ptype: PType, -} - pub(super) const SLOT_NAMES: [&str; 0] = []; #[derive(Clone, Debug)] @@ -76,8 +68,14 @@ pub struct SequenceData { pub(super) slots: Vec>, } +pub struct SequenceDataParts { + pub base: PValue, + pub multiplier: PValue, + pub ptype: PType, +} + impl SequenceData { - pub fn try_new_typed>( + pub(crate) fn try_new_typed>( base: T, multiplier: T, nullability: Nullability, @@ -93,7 +91,7 @@ impl SequenceData { } /// Constructs a sequence array using two integer values (with the same ptype). - pub fn try_new( + pub(crate) fn try_new( base: PValue, multiplier: PValue, ptype: PType, @@ -167,6 +165,14 @@ impl SequenceData { self.multiplier } + pub fn into_parts(self) -> SequenceDataParts { + SequenceDataParts { + base: self.base, + multiplier: self.multiplier, + ptype: self.base.ptype(), + } + } + pub(crate) fn try_last( base: PValue, multiplier: PValue, @@ -200,13 +206,6 @@ impl SequenceData { }) } - pub fn into_parts(self) -> SequenceArrayParts { - SequenceArrayParts { - base: self.base, - multiplier: self.multiplier, - ptype: self.base.ptype(), - } - } } impl VTable for Sequence { @@ -374,7 +373,7 @@ pub struct Sequence; impl Sequence { pub const ID: ArrayId = ArrayId::new_ref("vortex.sequence"); - fn stats(multiplier: PValue) -> ArrayStats { + fn stats(multiplier: PValue) -> StatsSet { // A sequence A[i] = base + i * multiplier is sorted iff multiplier >= 0, // and strictly sorted iff multiplier > 0. let (is_sorted, is_strict_sorted) = match_each_pvalue!( @@ -394,7 +393,7 @@ impl Sequence { ), ]) }; - ArrayStats::from(stats_set) + stats_set } /// Construct a new [`SequenceArray`] from pre-validated parts. @@ -414,7 +413,8 @@ impl Sequence { .vortex_expect("SequenceArray parts must be normalized to the target ptype"); let stats = Self::stats(multiplier); let data = unsafe { SequenceData::new_unchecked(base, multiplier) }; - Array::try_from_parts(ArrayParts::new(Sequence, dtype, length, data).with_stats(stats)) + Array::try_from_parts(ArrayParts::new(Sequence, dtype, length, data)) + .map(|array| array.with_stats_set(stats)) .vortex_expect("pre-validated SequenceArray parts must be valid") } @@ -429,7 +429,8 @@ impl Sequence { let dtype = DType::Primitive(ptype, nullability); let data = SequenceData::try_new(base, multiplier, ptype, nullability, length)?; let stats = Self::stats(data.multiplier()); - Array::try_from_parts(ArrayParts::new(Sequence, dtype, length, data).with_stats(stats)) + Array::try_from_parts(ArrayParts::new(Sequence, dtype, length, data)) + .map(|array| array.with_stats_set(stats)) } /// Construct a new typed [`SequenceArray`] from base/multiplier values. @@ -443,7 +444,8 @@ impl Sequence { let dtype = DType::Primitive(ptype, nullability); let data = SequenceData::try_new_typed(base, multiplier, nullability, length)?; let stats = Self::stats(data.multiplier()); - Array::try_from_parts(ArrayParts::new(Sequence, dtype, length, data).with_stats(stats)) + Array::try_from_parts(ArrayParts::new(Sequence, dtype, length, data)) + .map(|array| array.with_stats_set(stats)) } } diff --git a/encodings/sequence/src/lib.rs b/encodings/sequence/src/lib.rs index 958bbf6352f..a9505d8178a 100644 --- a/encodings/sequence/src/lib.rs +++ b/encodings/sequence/src/lib.rs @@ -13,8 +13,8 @@ pub use array::Sequence; /// Represents the equation A\[i\] = a * i + b. /// This can be used for compression, fast comparisons and also for row ids. pub use array::SequenceArray; -pub use array::SequenceArrayParts; pub use array::SequenceData; +pub use array::SequenceDataParts; pub use compress::sequence_encode; use vortex_array::aggregate_fn::AggregateFnVTable; use vortex_array::aggregate_fn::fns::is_sorted::IsSorted; diff --git a/encodings/zstd/public-api.lock b/encodings/zstd/public-api.lock index 85f0e4ebb05..dc7e1e63809 100644 --- a/encodings/zstd/public-api.lock +++ b/encodings/zstd/public-api.lock @@ -14,8 +14,6 @@ pub fn vortex_zstd::Zstd::from_var_bin_view(vbv: &vortex_array::arrays::varbinvi pub fn vortex_zstd::Zstd::from_var_bin_view_without_dict(vbv: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult -pub fn vortex_zstd::Zstd::into_parts(array: vortex_zstd::ZstdArray) -> vortex_zstd::ZstdArrayParts - pub fn vortex_zstd::Zstd::try_new(dtype: vortex_array::dtype::DType, data: vortex_zstd::ZstdData) -> vortex_error::VortexResult impl core::clone::Clone for vortex_zstd::Zstd @@ -74,28 +72,6 @@ impl vortex_array::scalar_fn::fns::cast::kernel::CastReduce for vortex_zstd::Zst pub fn vortex_zstd::Zstd::cast(array: vortex_array::array::view::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> -pub struct vortex_zstd::ZstdArrayParts - -pub vortex_zstd::ZstdArrayParts::dictionary: core::option::Option - -pub vortex_zstd::ZstdArrayParts::dtype: vortex_array::dtype::DType - -pub vortex_zstd::ZstdArrayParts::frames: alloc::vec::Vec - -pub vortex_zstd::ZstdArrayParts::metadata: vortex_zstd::ZstdMetadata - -pub vortex_zstd::ZstdArrayParts::n_rows: usize - -pub vortex_zstd::ZstdArrayParts::slice_start: usize - -pub vortex_zstd::ZstdArrayParts::slice_stop: usize - -pub vortex_zstd::ZstdArrayParts::validity: vortex_array::validity::Validity - -impl core::fmt::Debug for vortex_zstd::ZstdArrayParts - -pub fn vortex_zstd::ZstdArrayParts::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub struct vortex_zstd::ZstdBuffersMetadata pub vortex_zstd::ZstdBuffersMetadata::buffer_alignments: alloc::vec::Vec @@ -140,6 +116,8 @@ pub fn vortex_zstd::ZstdData::from_var_bin_view(vbv: &vortex_array::arrays::varb pub fn vortex_zstd::ZstdData::from_var_bin_view_without_dict(vbv: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult +pub fn vortex_zstd::ZstdData::into_parts(self) -> vortex_zstd::ZstdDataParts + pub fn vortex_zstd::ZstdData::is_empty(&self) -> bool pub fn vortex_zstd::ZstdData::len(&self) -> usize @@ -160,6 +138,22 @@ impl vortex_array::array::vtable::validity::ValiditySliceHelper for vortex_zstd: pub fn vortex_zstd::ZstdData::unsliced_validity_and_slice(&self) -> (&vortex_array::validity::Validity, usize, usize) +pub struct vortex_zstd::ZstdDataParts + +pub vortex_zstd::ZstdDataParts::dictionary: core::option::Option + +pub vortex_zstd::ZstdDataParts::frames: alloc::vec::Vec + +pub vortex_zstd::ZstdDataParts::metadata: vortex_zstd::ZstdMetadata + +pub vortex_zstd::ZstdDataParts::n_rows: usize + +pub vortex_zstd::ZstdDataParts::slice_start: usize + +pub vortex_zstd::ZstdDataParts::slice_stop: usize + +pub vortex_zstd::ZstdDataParts::validity: vortex_array::validity::Validity + pub struct vortex_zstd::ZstdFrameMetadata pub vortex_zstd::ZstdFrameMetadata::n_values: u64 diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index b0ce52a761d..46184ae8e2f 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -308,21 +308,6 @@ impl Zstd { array.data().decompress(array.dtype(), ctx) } - pub fn into_parts(array: ZstdArray) -> ZstdArrayParts { - let dtype = array.dtype().clone(); - let data = array.into_data(); - - ZstdArrayParts { - dictionary: data.dictionary, - frames: data.frames, - metadata: data.metadata, - dtype, - validity: data.unsliced_validity, - n_rows: data.unsliced_n_rows, - slice_start: data.slice_start, - slice_stop: data.slice_stop, - } - } } /// The validity bitmap indicating which elements are non-null. @@ -342,27 +327,18 @@ pub struct ZstdData { slice_stop: usize, } -/// The parts of a [`ZstdArray`] returned by [`ZstdArray::into_parts`]. -#[derive(Debug)] -pub struct ZstdArrayParts { - /// The optional dictionary used for compression. +pub struct ZstdDataParts { pub dictionary: Option, - /// The compressed frames. pub frames: Vec, - /// The compression metadata. pub metadata: ZstdMetadata, - /// The data type of the uncompressed array. - pub dtype: DType, - /// The validity of the uncompressed array. pub validity: Validity, - /// The number of rows in the uncompressed array. pub n_rows: usize, - /// Slice start offset. pub slice_start: usize, - /// Slice stop offset. pub slice_stop: usize, } +/// The parts of a [`ZstdArray`] returned by [`ZstdArray::into_parts`]. +#[derive(Debug)] struct Frames { dictionary: Option, frames: Vec, @@ -1013,6 +989,18 @@ impl ZstdData { self.slice_stop == self.slice_start } + pub fn into_parts(self) -> ZstdDataParts { + ZstdDataParts { + dictionary: self.dictionary, + frames: self.frames, + metadata: self.metadata, + validity: self.unsliced_validity, + n_rows: self.unsliced_n_rows, + slice_start: self.slice_start, + slice_stop: self.slice_stop, + } + } + pub(crate) fn slice_start(&self) -> usize { self.slice_start } diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 68d709c8e51..e4ead21dddb 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -888,6 +888,8 @@ pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ct pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Bool::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -942,16 +944,6 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::Bo pub fn vortex_array::arrays::Bool::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::Bool>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::bool::BoolArrayParts - -pub vortex_array::arrays::bool::BoolArrayParts::bits: vortex_array::buffer::BufferHandle - -pub vortex_array::arrays::bool::BoolArrayParts::len: usize - -pub vortex_array::arrays::bool::BoolArrayParts::offset: usize - -pub vortex_array::arrays::bool::BoolArrayParts::validity: vortex_array::validity::Validity - pub struct vortex_array::arrays::bool::BoolData impl vortex_array::arrays::bool::BoolData @@ -960,7 +952,7 @@ pub fn vortex_array::arrays::bool::BoolData::dtype(&self) -> vortex_array::dtype pub fn vortex_array::arrays::bool::BoolData::into_bit_buffer(self) -> vortex_buffer::bit::buf::BitBuffer -pub fn vortex_array::arrays::bool::BoolData::into_parts(self) -> vortex_array::arrays::bool::BoolArrayParts +pub fn vortex_array::arrays::bool::BoolData::into_parts(self) -> vortex_array::arrays::bool::BoolDataParts pub fn vortex_array::arrays::bool::BoolData::is_empty(&self) -> bool @@ -986,6 +978,16 @@ impl core::fmt::Debug for vortex_array::arrays::bool::BoolData pub fn vortex_array::arrays::bool::BoolData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct vortex_array::arrays::bool::BoolDataParts + +pub vortex_array::arrays::bool::BoolDataParts::bits: vortex_array::buffer::BufferHandle + +pub vortex_array::arrays::bool::BoolDataParts::len: usize + +pub vortex_array::arrays::bool::BoolDataParts::offset: usize + +pub vortex_array::arrays::bool::BoolDataParts::validity: vortex_array::validity::Validity + pub struct vortex_array::arrays::bool::BoolMaskedValidityRule impl core::default::Default for vortex_array::arrays::bool::BoolMaskedValidityRule @@ -1052,6 +1054,8 @@ pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Chunked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1200,6 +1204,8 @@ pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Constant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1380,6 +1386,8 @@ pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Decimal::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1434,16 +1442,6 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::De pub fn vortex_array::arrays::Decimal::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::decimal::DecimalArrayParts - -pub vortex_array::arrays::decimal::DecimalArrayParts::decimal_dtype: vortex_array::dtype::DecimalDType - -pub vortex_array::arrays::decimal::DecimalArrayParts::validity: vortex_array::validity::Validity - -pub vortex_array::arrays::decimal::DecimalArrayParts::values: vortex_array::buffer::BufferHandle - -pub vortex_array::arrays::decimal::DecimalArrayParts::values_type: vortex_array::dtype::DecimalType - pub struct vortex_array::arrays::decimal::DecimalData impl vortex_array::arrays::decimal::DecimalData @@ -1460,7 +1458,7 @@ pub fn vortex_array::arrays::decimal::DecimalData::from_iter>>(iter: I, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self -pub fn vortex_array::arrays::decimal::DecimalData::into_parts(self) -> vortex_array::arrays::decimal::DecimalArrayParts +pub fn vortex_array::arrays::decimal::DecimalData::into_parts(self) -> vortex_array::arrays::decimal::DecimalDataParts pub fn vortex_array::arrays::decimal::DecimalData::is_empty(&self) -> bool @@ -1498,6 +1496,16 @@ impl core::fmt::Debug for vortex_array::arrays::decimal::DecimalData pub fn vortex_array::arrays::decimal::DecimalData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct vortex_array::arrays::decimal::DecimalDataParts + +pub vortex_array::arrays::decimal::DecimalDataParts::decimal_dtype: vortex_array::dtype::DecimalDType + +pub vortex_array::arrays::decimal::DecimalDataParts::validity: vortex_array::validity::Validity + +pub vortex_array::arrays::decimal::DecimalDataParts::values: vortex_array::buffer::BufferHandle + +pub vortex_array::arrays::decimal::DecimalDataParts::values_type: vortex_array::dtype::DecimalType + pub struct vortex_array::arrays::decimal::DecimalMaskedValidityRule impl core::default::Default for vortex_array::arrays::decimal::DecimalMaskedValidityRule @@ -1568,6 +1576,8 @@ pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1672,6 +1682,8 @@ pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1728,12 +1740,6 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::di pub fn vortex_array::arrays::dict::Dict::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::dict::Dict>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::dict::DictArrayParts - -pub vortex_array::arrays::dict::DictArrayParts::codes: vortex_array::ArrayRef - -pub vortex_array::arrays::dict::DictArrayParts::values: vortex_array::ArrayRef - pub struct vortex_array::arrays::dict::DictData impl vortex_array::arrays::dict::DictData @@ -1744,7 +1750,7 @@ pub fn vortex_array::arrays::dict::DictData::dtype(&self) -> vortex_array::dtype pub fn vortex_array::arrays::dict::DictData::has_all_values_referenced(&self) -> bool -pub fn vortex_array::arrays::dict::DictData::into_parts(self) -> vortex_array::arrays::dict::DictArrayParts +pub fn vortex_array::arrays::dict::DictData::into_parts(self) -> vortex_array::arrays::dict::DictDataParts pub fn vortex_array::arrays::dict::DictData::is_empty(&self) -> bool @@ -1756,8 +1762,6 @@ pub unsafe fn vortex_array::arrays::dict::DictData::new_unchecked(codes: vortex_ pub unsafe fn vortex_array::arrays::dict::DictData::set_all_values_referenced(self, all_values_referenced: bool) -> Self -pub fn vortex_array::arrays::dict::DictData::try_new(codes: vortex_array::ArrayRef, values: vortex_array::ArrayRef) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::DictData::validate_all_values_referenced(&self) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::dict::DictData::values(&self) -> &vortex_array::ArrayRef @@ -1774,6 +1778,12 @@ impl core::fmt::Debug for vortex_array::arrays::dict::DictData pub fn vortex_array::arrays::dict::DictData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct vortex_array::arrays::dict::DictDataParts + +pub vortex_array::arrays::dict::DictDataParts::codes: vortex_array::ArrayRef + +pub vortex_array::arrays::dict::DictDataParts::values: vortex_array::ArrayRef + pub struct vortex_array::arrays::dict::DictMetadata impl vortex_array::arrays::dict::DictMetadata @@ -1960,6 +1970,8 @@ pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Extension::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2086,6 +2098,8 @@ pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Filter::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2102,7 +2116,7 @@ pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -2110,12 +2124,6 @@ impl vortex_array::ValidityVTable for vortex_array pub fn vortex_array::arrays::Filter::validity(array: vortex_array::ArrayView<'_, vortex_array::arrays::Filter>) -> vortex_error::VortexResult -pub struct vortex_array::arrays::filter::FilterArrayParts - -pub vortex_array::arrays::filter::FilterArrayParts::child: vortex_array::ArrayRef - -pub vortex_array::arrays::filter::FilterArrayParts::mask: vortex_mask::Mask - pub struct vortex_array::arrays::filter::FilterData impl vortex_array::arrays::filter::FilterData @@ -2126,18 +2134,14 @@ pub fn vortex_array::arrays::filter::FilterData::dtype(&self) -> &vortex_array:: pub fn vortex_array::arrays::filter::FilterData::filter_mask(&self) -> &vortex_mask::Mask +pub fn vortex_array::arrays::filter::FilterData::into_parts(self) -> vortex_array::arrays::filter::FilterDataParts + pub fn vortex_array::arrays::filter::FilterData::is_empty(&self) -> bool pub fn vortex_array::arrays::filter::FilterData::len(&self) -> usize pub fn vortex_array::arrays::filter::FilterData::new(array: vortex_array::ArrayRef, mask: vortex_mask::Mask) -> Self -pub fn vortex_array::arrays::filter::FilterData::try_new(array: vortex_array::ArrayRef, mask: vortex_mask::Mask) -> vortex_error::VortexResult - -impl vortex_array::arrays::filter::FilterData - -pub fn vortex_array::arrays::filter::FilterData::into_parts(self) -> vortex_array::arrays::filter::FilterArrayParts - impl core::clone::Clone for vortex_array::arrays::filter::FilterData pub fn vortex_array::arrays::filter::FilterData::clone(&self) -> vortex_array::arrays::filter::FilterData @@ -2146,6 +2150,12 @@ impl core::fmt::Debug for vortex_array::arrays::filter::FilterData pub fn vortex_array::arrays::filter::FilterData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct vortex_array::arrays::filter::FilterDataParts + +pub vortex_array::arrays::filter::FilterDataParts::child: vortex_array::ArrayRef + +pub vortex_array::arrays::filter::FilterDataParts::mask: vortex_mask::Mask + pub struct vortex_array::arrays::filter::FilterExecuteAdaptor(pub V) impl core::default::Default for vortex_array::arrays::filter::FilterExecuteAdaptor @@ -2276,6 +2286,8 @@ pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::FixedSizeList::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2336,8 +2348,6 @@ pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::new(elements: v pub unsafe fn vortex_array::arrays::fixed_size_list::FixedSizeListData::new_unchecked(elements: vortex_array::ArrayRef, list_size: u32, validity: vortex_array::validity::Validity, len: usize) -> Self -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::try_new(elements: vortex_array::ArrayRef, list_size: u32, validity: vortex_array::validity::Validity, len: usize) -> vortex_error::VortexResult - pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::validate(elements: &vortex_array::ArrayRef, len: usize, list_size: u32, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::validity(&self) -> vortex_array::validity::Validity @@ -2406,6 +2416,8 @@ pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::List::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2450,16 +2462,6 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::Li pub fn vortex_array::arrays::List::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::list::ListArrayParts - -pub vortex_array::arrays::list::ListArrayParts::dtype: vortex_array::dtype::DType - -pub vortex_array::arrays::list::ListArrayParts::elements: vortex_array::ArrayRef - -pub vortex_array::arrays::list::ListArrayParts::offsets: vortex_array::ArrayRef - -pub vortex_array::arrays::list::ListArrayParts::validity: vortex_array::validity::Validity - pub struct vortex_array::arrays::list::ListData impl vortex_array::arrays::list::ListData @@ -2470,7 +2472,7 @@ pub fn vortex_array::arrays::list::ListData::element_dtype(&self) -> &vortex_arr pub fn vortex_array::arrays::list::ListData::elements(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::list::ListData::into_parts(self) -> vortex_array::arrays::list::ListArrayParts +pub fn vortex_array::arrays::list::ListData::into_parts(self) -> vortex_array::arrays::list::ListDataParts pub fn vortex_array::arrays::list::ListData::is_empty(&self) -> bool @@ -2488,8 +2490,6 @@ pub fn vortex_array::arrays::list::ListData::offsets(&self) -> &vortex_array::Ar pub fn vortex_array::arrays::list::ListData::sliced_elements(&self) -> vortex_error::VortexResult -pub fn vortex_array::arrays::list::ListData::try_new(elements: vortex_array::ArrayRef, offsets: vortex_array::ArrayRef, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult - pub fn vortex_array::arrays::list::ListData::validate(elements: &vortex_array::ArrayRef, offsets: &vortex_array::ArrayRef, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::list::ListData::validity(&self) -> vortex_array::validity::Validity @@ -2508,6 +2508,16 @@ impl core::fmt::Debug for vortex_array::arrays::list::ListData pub fn vortex_array::arrays::list::ListData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct vortex_array::arrays::list::ListDataParts + +pub vortex_array::arrays::list::ListDataParts::dtype: vortex_array::dtype::DType + +pub vortex_array::arrays::list::ListDataParts::elements: vortex_array::ArrayRef + +pub vortex_array::arrays::list::ListDataParts::offsets: vortex_array::ArrayRef + +pub vortex_array::arrays::list::ListDataParts::validity: vortex_array::validity::Validity + pub type vortex_array::arrays::list::ListArray = vortex_array::Array pub mod vortex_array::arrays::listview @@ -2568,6 +2578,8 @@ pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::ListView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2608,18 +2620,6 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::Li pub fn vortex_array::arrays::ListView::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::listview::ListViewArrayParts - -pub vortex_array::arrays::listview::ListViewArrayParts::elements: vortex_array::ArrayRef - -pub vortex_array::arrays::listview::ListViewArrayParts::elements_dtype: alloc::sync::Arc - -pub vortex_array::arrays::listview::ListViewArrayParts::offsets: vortex_array::ArrayRef - -pub vortex_array::arrays::listview::ListViewArrayParts::sizes: vortex_array::ArrayRef - -pub vortex_array::arrays::listview::ListViewArrayParts::validity: vortex_array::validity::Validity - pub struct vortex_array::arrays::listview::ListViewData impl vortex_array::arrays::listview::ListViewData @@ -2628,7 +2628,7 @@ pub fn vortex_array::arrays::listview::ListViewData::dtype(&self) -> vortex_arra pub fn vortex_array::arrays::listview::ListViewData::elements(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::listview::ListViewData::into_parts(self) -> vortex_array::arrays::listview::ListViewArrayParts +pub fn vortex_array::arrays::listview::ListViewData::into_parts(self) -> vortex_array::arrays::listview::ListViewDataParts pub fn vortex_array::arrays::listview::ListViewData::is_empty(&self) -> bool @@ -2670,6 +2670,18 @@ impl core::fmt::Debug for vortex_array::arrays::listview::ListViewData pub fn vortex_array::arrays::listview::ListViewData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct vortex_array::arrays::listview::ListViewDataParts + +pub vortex_array::arrays::listview::ListViewDataParts::elements: vortex_array::ArrayRef + +pub vortex_array::arrays::listview::ListViewDataParts::elements_dtype: alloc::sync::Arc + +pub vortex_array::arrays::listview::ListViewDataParts::offsets: vortex_array::ArrayRef + +pub vortex_array::arrays::listview::ListViewDataParts::sizes: vortex_array::ArrayRef + +pub vortex_array::arrays::listview::ListViewDataParts::validity: vortex_array::validity::Validity + pub fn vortex_array::arrays::listview::list_from_list_view(list_view: vortex_array::arrays::ListViewArray) -> vortex_error::VortexResult pub fn vortex_array::arrays::listview::list_view_from_list(list: vortex_array::arrays::ListArray, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -2726,6 +2738,8 @@ pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Masked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2778,8 +2792,6 @@ pub fn vortex_array::arrays::masked::MaskedData::is_empty(&self) -> bool pub fn vortex_array::arrays::masked::MaskedData::len(&self) -> usize -pub fn vortex_array::arrays::masked::MaskedData::try_new(child: vortex_array::ArrayRef, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult - pub fn vortex_array::arrays::masked::MaskedData::validity(&self) -> vortex_array::validity::Validity pub fn vortex_array::arrays::masked::MaskedData::validity_mask(&self) -> vortex_mask::Mask @@ -2848,6 +2860,8 @@ pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::null::Null::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2952,6 +2966,8 @@ pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::patched::Patched::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3166,6 +3182,8 @@ pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Primitive::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3220,14 +3238,6 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::Pr pub fn vortex_array::arrays::Primitive::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::primitive::PrimitiveArrayParts - -pub vortex_array::arrays::primitive::PrimitiveArrayParts::buffer: vortex_array::buffer::BufferHandle - -pub vortex_array::arrays::primitive::PrimitiveArrayParts::ptype: vortex_array::dtype::PType - -pub vortex_array::arrays::primitive::PrimitiveArrayParts::validity: vortex_array::validity::Validity - pub struct vortex_array::arrays::primitive::PrimitiveData impl vortex_array::arrays::primitive::PrimitiveData @@ -3286,7 +3296,7 @@ pub fn vortex_array::arrays::primitive::PrimitiveData::validate vortex_array::arrays::primitive::PrimitiveArrayParts +pub fn vortex_array::arrays::primitive::PrimitiveData::into_parts(self) -> vortex_array::arrays::primitive::PrimitiveDataParts impl core::clone::Clone for vortex_array::arrays::primitive::PrimitiveData @@ -3296,6 +3306,14 @@ impl core::fmt::Debug for vortex_array::arrays::primitive::PrimitiveData pub fn vortex_array::arrays::primitive::PrimitiveData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct vortex_array::arrays::primitive::PrimitiveDataParts + +pub vortex_array::arrays::primitive::PrimitiveDataParts::buffer: vortex_array::buffer::BufferHandle + +pub vortex_array::arrays::primitive::PrimitiveDataParts::ptype: vortex_array::dtype::PType + +pub vortex_array::arrays::primitive::PrimitiveDataParts::validity: vortex_array::validity::Validity + pub struct vortex_array::arrays::primitive::PrimitiveMaskedValidityRule impl core::default::Default for vortex_array::arrays::primitive::PrimitiveMaskedValidityRule @@ -3430,6 +3448,8 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3512,6 +3532,8 @@ pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Shared::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3610,6 +3632,8 @@ pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::slice::Slice::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3626,7 +3650,7 @@ pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::Array pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3638,12 +3662,6 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::slice::S pub fn vortex_array::arrays::slice::Slice::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::slice::SliceArrayParts - -pub vortex_array::arrays::slice::SliceArrayParts::child: vortex_array::ArrayRef - -pub vortex_array::arrays::slice::SliceArrayParts::range: core::ops::range::Range - pub struct vortex_array::arrays::slice::SliceData impl vortex_array::arrays::slice::SliceData @@ -3652,6 +3670,8 @@ pub fn vortex_array::arrays::slice::SliceData::child(&self) -> &vortex_array::Ar pub fn vortex_array::arrays::slice::SliceData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::slice::SliceData::into_parts(self) -> vortex_array::arrays::slice::SliceDataParts + pub fn vortex_array::arrays::slice::SliceData::is_empty(&self) -> bool pub fn vortex_array::arrays::slice::SliceData::len(&self) -> usize @@ -3660,12 +3680,6 @@ pub fn vortex_array::arrays::slice::SliceData::new(child: vortex_array::ArrayRef pub fn vortex_array::arrays::slice::SliceData::slice_range(&self) -> &core::ops::range::Range -pub fn vortex_array::arrays::slice::SliceData::try_new(child: vortex_array::ArrayRef, range: core::ops::range::Range) -> vortex_error::VortexResult - -impl vortex_array::arrays::slice::SliceData - -pub fn vortex_array::arrays::slice::SliceData::into_parts(self) -> vortex_array::arrays::slice::SliceArrayParts - impl core::clone::Clone for vortex_array::arrays::slice::SliceData pub fn vortex_array::arrays::slice::SliceData::clone(&self) -> vortex_array::arrays::slice::SliceData @@ -3674,6 +3688,12 @@ impl core::fmt::Debug for vortex_array::arrays::slice::SliceData pub fn vortex_array::arrays::slice::SliceData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct vortex_array::arrays::slice::SliceDataParts + +pub vortex_array::arrays::slice::SliceDataParts::child: vortex_array::ArrayRef + +pub vortex_array::arrays::slice::SliceDataParts::range: core::ops::range::Range + pub struct vortex_array::arrays::slice::SliceExecuteAdaptor(pub V) impl core::default::Default for vortex_array::arrays::slice::SliceExecuteAdaptor @@ -3838,6 +3858,8 @@ pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Struct::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3882,14 +3904,6 @@ impl vortex_array::scalar_fn::fns::zip::ZipKernel for vortex_array::arrays::Stru pub fn vortex_array::arrays::Struct::zip(if_true: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, if_false: &vortex_array::ArrayRef, mask: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::struct_::StructArrayParts - -pub vortex_array::arrays::struct_::StructArrayParts::fields: alloc::sync::Arc<[vortex_array::ArrayRef]> - -pub vortex_array::arrays::struct_::StructArrayParts::struct_fields: vortex_array::dtype::StructFields - -pub vortex_array::arrays::struct_::StructArrayParts::validity: vortex_array::validity::Validity - pub struct vortex_array::arrays::struct_::StructData impl vortex_array::arrays::struct_::StructData @@ -3900,7 +3914,7 @@ pub fn vortex_array::arrays::struct_::StructData::from_fields alloc::vec::Vec -pub fn vortex_array::arrays::struct_::StructData::into_parts(self) -> vortex_array::arrays::struct_::StructArrayParts +pub fn vortex_array::arrays::struct_::StructData::into_parts(self) -> vortex_array::arrays::struct_::StructDataParts pub fn vortex_array::arrays::struct_::StructData::is_empty(&self) -> bool @@ -3926,10 +3940,6 @@ pub fn vortex_array::arrays::struct_::StructData::try_from_iter, A: vortex_array::IntoArray, T: core::iter::traits::collect::IntoIterator>(iter: T, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult -pub fn vortex_array::arrays::struct_::StructData::try_new(names: vortex_array::dtype::FieldNames, fields: impl core::convert::Into>, length: usize, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::struct_::StructData::try_new_with_dtype(fields: impl core::convert::Into>, dtype: vortex_array::dtype::StructFields, length: usize, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult - pub fn vortex_array::arrays::struct_::StructData::unmasked_field(&self, idx: usize) -> &vortex_array::ArrayRef pub fn vortex_array::arrays::struct_::StructData::unmasked_field_by_name(&self, name: impl core::convert::AsRef) -> vortex_error::VortexResult<&vortex_array::ArrayRef> @@ -3956,6 +3966,14 @@ impl core::fmt::Debug for vortex_array::arrays::struct_::StructData pub fn vortex_array::arrays::struct_::StructData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct vortex_array::arrays::struct_::StructDataParts + +pub vortex_array::arrays::struct_::StructDataParts::fields: alloc::sync::Arc<[vortex_array::ArrayRef]> + +pub vortex_array::arrays::struct_::StructDataParts::struct_fields: vortex_array::dtype::StructFields + +pub vortex_array::arrays::struct_::StructDataParts::validity: vortex_array::validity::Validity + pub type vortex_array::arrays::struct_::StructArray = vortex_array::Array pub mod vortex_array::arrays::varbin @@ -4036,6 +4054,8 @@ pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::VarBin::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -4426,6 +4446,8 @@ pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::VarBinView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -4470,16 +4492,6 @@ impl vortex_array::scalar_fn::fns::zip::ZipKernel for vortex_array::arrays::VarB pub fn vortex_array::arrays::VarBinView::zip(if_true: vortex_array::ArrayView<'_, vortex_array::arrays::VarBinView>, if_false: &vortex_array::ArrayRef, mask: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::varbinview::VarBinViewArrayParts - -pub vortex_array::arrays::varbinview::VarBinViewArrayParts::buffers: alloc::sync::Arc<[vortex_array::buffer::BufferHandle]> - -pub vortex_array::arrays::varbinview::VarBinViewArrayParts::dtype: vortex_array::dtype::DType - -pub vortex_array::arrays::varbinview::VarBinViewArrayParts::validity: vortex_array::validity::Validity - -pub vortex_array::arrays::varbinview::VarBinViewArrayParts::views: vortex_array::buffer::BufferHandle - pub struct vortex_array::arrays::varbinview::VarBinViewData impl vortex_array::arrays::varbinview::VarBinViewData @@ -4502,7 +4514,7 @@ pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter_nullable_str< pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter_str, I: core::iter::traits::collect::IntoIterator>(iter: I) -> Self -pub fn vortex_array::arrays::varbinview::VarBinViewData::into_parts(self) -> vortex_array::arrays::varbinview::VarBinViewArrayParts +pub fn vortex_array::arrays::varbinview::VarBinViewData::into_parts(self) -> vortex_array::arrays::varbinview::VarBinViewDataParts pub fn vortex_array::arrays::varbinview::VarBinViewData::is_empty(&self) -> bool @@ -4556,6 +4568,16 @@ impl<'a> core::iter::traits::collect::FromIterator pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter>>(iter: T) -> Self +pub struct vortex_array::arrays::varbinview::VarBinViewDataParts + +pub vortex_array::arrays::varbinview::VarBinViewDataParts::buffers: alloc::sync::Arc<[vortex_array::buffer::BufferHandle]> + +pub vortex_array::arrays::varbinview::VarBinViewDataParts::dtype: vortex_array::dtype::DType + +pub vortex_array::arrays::varbinview::VarBinViewDataParts::validity: vortex_array::validity::Validity + +pub vortex_array::arrays::varbinview::VarBinViewDataParts::views: vortex_array::buffer::BufferHandle + pub type vortex_array::arrays::varbinview::VarBinViewArray = vortex_array::Array pub mod vortex_array::arrays::variant @@ -4606,6 +4628,8 @@ pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Variant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -4622,7 +4646,7 @@ pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -4636,7 +4660,7 @@ impl vortex_array::arrays::variant::VariantData pub fn vortex_array::arrays::variant::VariantData::child(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::variant::VariantData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::variant::VariantData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::variant::VariantData::is_empty(&self) -> bool @@ -4700,6 +4724,8 @@ pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ct pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Bool::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -4800,6 +4826,8 @@ pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Chunked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -4902,6 +4930,8 @@ pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Constant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5000,6 +5030,8 @@ pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Decimal::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5100,6 +5132,8 @@ pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5202,6 +5236,8 @@ pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Extension::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5296,6 +5332,8 @@ pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Filter::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5312,7 +5350,7 @@ pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5366,6 +5404,8 @@ pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::FixedSizeList::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5452,6 +5492,8 @@ pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::List::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5542,6 +5584,8 @@ pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::ListView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5628,6 +5672,8 @@ pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Masked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5718,6 +5764,8 @@ pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::null::Null::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5804,6 +5852,8 @@ pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::patched::Patched::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5930,6 +5980,8 @@ pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Primitive::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6026,6 +6078,8 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6096,6 +6150,8 @@ pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Shared::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6166,6 +6222,8 @@ pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::slice::Slice::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6182,7 +6240,7 @@ pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::Array pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6240,6 +6298,8 @@ pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Struct::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6334,6 +6394,8 @@ pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::VarBin::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6428,6 +6490,8 @@ pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::VarBinView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6518,6 +6582,8 @@ pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Variant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6534,7 +6600,7 @@ pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -18752,6 +18818,8 @@ pub fn vortex_array::vtable::ArrayVTable::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::vtable::ArrayVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::vtable::ArrayVTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::vtable::ArrayVTable::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -18768,7 +18836,7 @@ pub fn vortex_array::vtable::ArrayVTable::slot_name(array: vortex_array::ArrayVi pub fn vortex_array::vtable::ArrayVTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] -pub fn vortex_array::vtable::ArrayVTable::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::vtable::ArrayVTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::vtable::ArrayVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -18800,6 +18868,8 @@ pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ct pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Bool::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -18848,6 +18918,8 @@ pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Chunked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -18896,6 +18968,8 @@ pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Constant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -18944,6 +19018,8 @@ pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Decimal::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -18992,6 +19068,8 @@ pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Extension::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19040,6 +19118,8 @@ pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Filter::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19056,7 +19136,7 @@ pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19088,6 +19168,8 @@ pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::FixedSizeList::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19136,6 +19218,8 @@ pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::List::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19184,6 +19268,8 @@ pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::ListView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19232,6 +19318,8 @@ pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Masked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19280,6 +19368,8 @@ pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Primitive::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19328,6 +19418,8 @@ pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Shared::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19376,6 +19468,8 @@ pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Struct::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19424,6 +19518,8 @@ pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::VarBin::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19472,6 +19568,8 @@ pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::VarBinView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19520,6 +19618,8 @@ pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Variant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19536,7 +19636,7 @@ pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19568,6 +19668,8 @@ pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19616,6 +19718,8 @@ pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::null::Null::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19664,6 +19768,8 @@ pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::patched::Patched::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19712,6 +19818,8 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19760,6 +19868,8 @@ pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::slice::Slice::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19776,7 +19886,7 @@ pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::Array pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19932,6 +20042,8 @@ pub fn vortex_array::vtable::VTable::execute(array: vortex_array::Array, c pub fn vortex_array::vtable::VTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::vtable::VTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::vtable::VTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::vtable::VTable::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19948,7 +20060,7 @@ pub fn vortex_array::vtable::VTable::slot_name(array: vortex_array::ArrayView<'_ pub fn vortex_array::vtable::VTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] -pub fn vortex_array::vtable::VTable::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::vtable::VTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::vtable::VTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19980,6 +20092,8 @@ pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ct pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Bool::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20028,6 +20142,8 @@ pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Chunked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20076,6 +20192,8 @@ pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Constant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20124,6 +20242,8 @@ pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Decimal::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20172,6 +20292,8 @@ pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Extension::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20220,6 +20342,8 @@ pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Filter::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20236,7 +20360,7 @@ pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20268,6 +20392,8 @@ pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::FixedSizeList::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20316,6 +20442,8 @@ pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::List::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20364,6 +20492,8 @@ pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::ListView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20412,6 +20542,8 @@ pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Masked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20460,6 +20592,8 @@ pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Primitive::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20508,6 +20642,8 @@ pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Shared::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20556,6 +20692,8 @@ pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Struct::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20604,6 +20742,8 @@ pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::VarBin::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20652,6 +20792,8 @@ pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::VarBinView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20700,6 +20842,8 @@ pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Variant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20716,7 +20860,7 @@ pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20748,6 +20892,8 @@ pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20796,6 +20942,8 @@ pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::null::Null::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20844,6 +20992,8 @@ pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::patched::Patched::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20892,6 +21042,8 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20940,6 +21092,8 @@ pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::slice::Slice::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20956,7 +21110,7 @@ pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::Array pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21352,8 +21506,6 @@ pub fn vortex_array::Array::from_indices::into_bit_buffer(self) -> vortex_buffer::bit::buf::BitBuffer -pub fn vortex_array::Array::into_encoding_parts(self) -> vortex_array::arrays::bool::BoolArrayParts - pub fn vortex_array::Array::new(bits: vortex_buffer::bit::buf::BitBuffer, validity: vortex_array::validity::Validity) -> Self pub fn vortex_array::Array::new_handle(handle: vortex_array::buffer::BufferHandle, offset: usize, len: usize, validity: vortex_array::validity::Validity) -> Self @@ -21496,8 +21648,6 @@ impl vortex_array::Array pub fn vortex_array::Array::from_fields>(items: &[(N, vortex_array::ArrayRef)]) -> vortex_error::VortexResult -pub fn vortex_array::Array::into_encoding_parts(self) -> vortex_array::arrays::struct_::StructArrayParts - pub fn vortex_array::Array::new(names: vortex_array::dtype::FieldNames, fields: impl core::convert::Into>, length: usize, validity: vortex_array::validity::Validity) -> Self pub fn vortex_array::Array::new_fieldless_with_len(len: usize) -> Self @@ -21664,6 +21814,8 @@ pub fn vortex_array::Array::try_from_array_ref(array: vortex_array::ArrayRef) pub fn vortex_array::Array::try_from_parts(new: vortex_array::ArrayParts) -> vortex_error::VortexResult +pub fn vortex_array::Array::with_stats_set(self, stats: vortex_array::stats::StatsSet) -> Self + impl core::convert::From> for vortex_array::Array pub fn vortex_array::Array::from(value: alloc::vec::Vec<&[u8]>) -> Self @@ -21772,16 +21924,12 @@ pub vortex_array::ArrayParts::dtype: vortex_array::dtype::DType pub vortex_array::ArrayParts::len: usize -pub vortex_array::ArrayParts::stats: vortex_array::stats::ArrayStats - pub vortex_array::ArrayParts::vtable: V impl vortex_array::ArrayParts pub fn vortex_array::ArrayParts::new(vtable: V, dtype: vortex_array::dtype::DType, len: usize, data: ::ArrayData) -> Self -pub fn vortex_array::ArrayParts::with_stats(self, stats: vortex_array::stats::ArrayStats) -> Self - pub struct vortex_array::ArrayRef(_) impl vortex_array::ArrayRef @@ -21840,6 +21988,8 @@ pub fn vortex_array::ArrayRef::len(&self) -> usize pub fn vortex_array::ArrayRef::metadata(&self) -> vortex_error::VortexResult>> +pub fn vortex_array::ArrayRef::metadata_fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::ArrayRef::named_buffers(&self) -> alloc::vec::Vec<(alloc::string::String, vortex_array::buffer::BufferHandle)> pub fn vortex_array::ArrayRef::named_children(&self) -> alloc::vec::Vec<(alloc::string::String, vortex_array::ArrayRef)> @@ -22570,6 +22720,8 @@ pub fn vortex_array::ArrayVTable::execute(array: vortex_array::Array, ctx: pub fn vortex_array::ArrayVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::ArrayVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::ArrayVTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::ArrayVTable::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22586,7 +22738,7 @@ pub fn vortex_array::ArrayVTable::slot_name(array: vortex_array::ArrayView<'_, S pub fn vortex_array::ArrayVTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] -pub fn vortex_array::ArrayVTable::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::ArrayVTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::ArrayVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -22618,6 +22770,8 @@ pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ct pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Bool::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22666,6 +22820,8 @@ pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Chunked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22714,6 +22870,8 @@ pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Constant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22762,6 +22920,8 @@ pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Decimal::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22810,6 +22970,8 @@ pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Extension::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22858,6 +23020,8 @@ pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Filter::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22874,7 +23038,7 @@ pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -22906,6 +23070,8 @@ pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::FixedSizeList::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22954,6 +23120,8 @@ pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::List::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23002,6 +23170,8 @@ pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::ListView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23050,6 +23220,8 @@ pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Masked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23098,6 +23270,8 @@ pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Primitive::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23146,6 +23320,8 @@ pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Shared::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23194,6 +23370,8 @@ pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Struct::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23242,6 +23420,8 @@ pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::VarBin::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23290,6 +23470,8 @@ pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::VarBinView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23338,6 +23520,8 @@ pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Variant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23354,7 +23538,7 @@ pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23386,6 +23570,8 @@ pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23434,6 +23620,8 @@ pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::null::Null::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23482,6 +23670,8 @@ pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::patched::Patched::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23530,6 +23720,8 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23578,6 +23770,8 @@ pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::slice::Slice::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23594,7 +23788,7 @@ pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::Array pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23990,6 +24184,8 @@ pub fn vortex_array::VTable::execute(array: vortex_array::Array, ctx: &mut pub fn vortex_array::VTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::VTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::VTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::VTable::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24006,7 +24202,7 @@ pub fn vortex_array::VTable::slot_name(array: vortex_array::ArrayView<'_, Self>, pub fn vortex_array::VTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] -pub fn vortex_array::VTable::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::VTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::VTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24038,6 +24234,8 @@ pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ct pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Bool::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24086,6 +24284,8 @@ pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Chunked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24134,6 +24334,8 @@ pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Constant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24182,6 +24384,8 @@ pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Decimal::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24230,6 +24434,8 @@ pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Extension::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24278,6 +24484,8 @@ pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Filter::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24294,7 +24502,7 @@ pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24326,6 +24534,8 @@ pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::FixedSizeList::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24374,6 +24584,8 @@ pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::List::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24422,6 +24634,8 @@ pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::ListView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24470,6 +24684,8 @@ pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Masked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24518,6 +24734,8 @@ pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Primitive::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24566,6 +24784,8 @@ pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Shared::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24614,6 +24834,8 @@ pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Struct::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24662,6 +24884,8 @@ pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, c pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::VarBin::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24710,6 +24934,8 @@ pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::VarBinView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24758,6 +24984,8 @@ pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::Variant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24774,7 +25002,7 @@ pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24806,6 +25034,8 @@ pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24854,6 +25084,8 @@ pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::null::Null::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24902,6 +25134,8 @@ pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::patched::Patched::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24950,6 +25184,8 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24998,6 +25234,8 @@ pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::slice::Slice::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25014,7 +25252,7 @@ pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::Array pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::validate(&self, _data: &Self::ArrayData, _dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> diff --git a/vortex-array/src/array/erased.rs b/vortex-array/src/array/erased.rs index f2c1a7a50d5..bdb67697c3a 100644 --- a/vortex-array/src/array/erased.rs +++ b/vortex-array/src/array/erased.rs @@ -467,6 +467,11 @@ impl ArrayRef { self.0.metadata(self) } + /// Formats a human-readable metadata description. + pub fn metadata_fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.0.metadata_fmt(self, f) + } + /// Returns whether all buffers are host-resident. pub fn is_host(&self) -> bool { for array in self.depth_first_traversal() { diff --git a/vortex-array/src/array/mod.rs b/vortex-array/src/array/mod.rs index 91701e6fae2..0dbf065e6a1 100644 --- a/vortex-array/src/array/mod.rs +++ b/vortex-array/src/array/mod.rs @@ -3,6 +3,7 @@ use std::any::Any; use std::fmt::Debug; +use std::fmt::Formatter; use std::hash::Hash; use std::hash::Hasher; @@ -121,6 +122,9 @@ pub(crate) trait DynArray: 'static + private::Sealed + Send + Sync + Debug { /// support serialization. fn metadata(&self, this: &ArrayRef) -> VortexResult>>; + /// Formats a human-readable metadata description. + fn metadata_fmt(&self, this: &ArrayRef, f: &mut Formatter<'_>) -> std::fmt::Result; + /// Hashes the array contents including len, dtype, and encoding id. fn dyn_array_hash(&self, state: &mut dyn Hasher, precision: crate::Precision); @@ -303,6 +307,11 @@ impl DynArray for ArrayInner { V::serialize(view) } + fn metadata_fmt(&self, this: &ArrayRef, f: &mut Formatter<'_>) -> std::fmt::Result { + let view = unsafe { ArrayView::new_unchecked(this, &self.data) }; + V::fmt_metadata(view, f) + } + fn dyn_array_hash(&self, state: &mut dyn Hasher, precision: crate::Precision) { let mut wrapper = HasherWrapper(state); self.len.hash(&mut wrapper); diff --git a/vortex-array/src/array/typed.rs b/vortex-array/src/array/typed.rs index 85a6fb56fc5..ae846b71265 100644 --- a/vortex-array/src/array/typed.rs +++ b/vortex-array/src/array/typed.rs @@ -22,6 +22,7 @@ use crate::array::ArrayView; use crate::array::VTable; use crate::dtype::DType; use crate::stats::ArrayStats; +use crate::stats::StatsSet; use crate::stats::StatsSetRef; use crate::validity::Validity; @@ -31,7 +32,6 @@ pub struct ArrayParts { pub dtype: DType, pub len: usize, pub data: V::ArrayData, - pub stats: ArrayStats, } impl ArrayParts { @@ -41,15 +41,8 @@ impl ArrayParts { dtype, len, data, - stats: ArrayStats::default(), } } - - #[must_use] - pub fn with_stats(mut self, stats: ArrayStats) -> Self { - self.stats = stats; - self - } } // ============================================================================= // ArrayInner — the concrete type stored inside Arc @@ -75,7 +68,13 @@ impl ArrayInner { pub fn try_new(new: ArrayParts) -> VortexResult { new.vtable.validate(&new.data, &new.dtype, new.len)?; Ok(unsafe { - Self::from_data_unchecked(new.vtable, new.dtype, new.len, new.data, new.stats) + Self::from_data_unchecked( + new.vtable, + new.dtype, + new.len, + new.data, + ArrayStats::default(), + ) }) } @@ -181,9 +180,7 @@ impl Array { /// Caller must ensure the provided parts are logically consistent. pub(crate) unsafe fn from_parts_unchecked(new: ArrayParts) -> Self { let inner = ArrayRef::from_inner(Arc::new(unsafe { - ArrayInner::::from_data_unchecked( - new.vtable, new.dtype, new.len, new.data, new.stats, - ) + ArrayInner::::from_data_unchecked(new.vtable, new.dtype, new.len, new.data, ArrayStats::default()) })); Self { inner, @@ -253,10 +250,14 @@ impl Array { dtype: inner.dtype.clone(), len: inner.len, data: inner.data.clone(), - stats: inner.stats.clone(), } } + pub fn with_stats_set(self, stats: StatsSet) -> Self { + self.statistics().replace(stats); + self + } + /// Returns a clone of the inner encoding-specific data. pub fn into_data(self) -> V::ArrayData { self.downcast_inner().data.clone() diff --git a/vortex-array/src/array/vtable/dyn_.rs b/vortex-array/src/array/vtable/dyn_.rs index 8afb9f80f9c..63df8c2f880 100644 --- a/vortex-array/src/array/vtable/dyn_.rs +++ b/vortex-array/src/array/vtable/dyn_.rs @@ -22,8 +22,6 @@ use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::executor::ExecutionCtx; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; - /// Reference-counted DynVTable pub type DynVTableRef = Arc; @@ -86,11 +84,8 @@ impl DynVTable for V { session: &VortexSession, ) -> VortexResult { let inner = self.deserialize(dtype, len, metadata, buffers, children, session)?; - Ok(Array::::try_from_parts( - ArrayParts::new(self.clone(), dtype.clone(), len, inner) - .with_stats(ArrayStats::default()), - )? - .into_array()) + Ok(Array::::try_from_parts(ArrayParts::new(self.clone(), dtype.clone(), len, inner))? + .into_array()) } fn with_slots(&self, array: ArrayRef, slots: Vec>) -> VortexResult { @@ -99,11 +94,15 @@ impl DynVTable for V { .vortex_expect("Failed to downcast array"); let mut data = typed.data().clone(); V::with_slots(&mut data, slots)?; + let stats = array.statistics().to_owned(); Ok(unsafe { - Array::::from_parts_unchecked( - ArrayParts::new(self.clone(), array.dtype().clone(), array.len(), data) - .with_stats(array.statistics().to_array_stats()), - ) + Array::::from_parts_unchecked(ArrayParts::new( + self.clone(), + array.dtype().clone(), + array.len(), + data, + )) + .with_stats_set(stats) .into_array() }) } diff --git a/vortex-array/src/array/vtable/mod.rs b/vortex-array/src/array/vtable/mod.rs index 8a2f258697b..305cbef6ca2 100644 --- a/vortex-array/src/array/vtable/mod.rs +++ b/vortex-array/src/array/vtable/mod.rs @@ -8,6 +8,7 @@ mod operations; mod validity; use std::fmt::Debug; +use std::fmt::Formatter; use std::hash::Hasher; pub use dyn_::*; @@ -60,9 +61,7 @@ pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { fn id(&self) -> ArrayId; /// Validates that externally supplied logical metadata matches the array data. - fn validate(&self, _data: &Self::ArrayData, _dtype: &DType, _len: usize) -> VortexResult<()> { - Ok(()) - } + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()>; /// Hashes the array contents. fn array_hash(array: &Self::ArrayData, state: &mut H, precision: Precision); @@ -123,6 +122,14 @@ pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { /// Return `None` if the array cannot be serialized. fn serialize(array: ArrayView<'_, Self>) -> VortexResult>>; + /// Formats a human-readable metadata description for display tooling. + fn fmt_metadata(array: ArrayView<'_, Self>, f: &mut Formatter<'_>) -> std::fmt::Result { + match Self::serialize(array) { + Ok(Some(metadata)) if metadata.is_empty() => f.write_str("EmptyMetadata"), + _ => Debug::fmt(array.data(), f), + } + } + /// Deserialize an array from serialized components. fn deserialize( &self, diff --git a/vortex-array/src/arrays/bool/array.rs b/vortex-array/src/arrays/bool/array.rs index db22f04ecf1..ec06e4d1106 100644 --- a/vortex-array/src/arrays/bool/array.rs +++ b/vortex-array/src/arrays/bool/array.rs @@ -69,7 +69,7 @@ pub struct BoolData { pub(super) len: usize, } -pub struct BoolArrayParts { +pub struct BoolDataParts { pub bits: BufferHandle, pub offset: usize, pub len: usize, @@ -118,9 +118,9 @@ impl BoolData { /// Splits into owned parts #[inline] - pub fn into_parts(self) -> BoolArrayParts { + pub fn into_parts(self) -> BoolDataParts { let validity = self.validity(); - BoolArrayParts { + BoolDataParts { bits: self.bits, offset: self.offset, len: self.len, @@ -242,11 +242,6 @@ impl Array { self.into_data().into_bit_buffer() } - /// Splits into the encoding-specific owned parts, consuming self. - #[inline] - pub fn into_encoding_parts(self) -> BoolArrayParts { - self.into_data().into_parts() - } } /// Internal constructors on BoolData (used by Array constructors and VTable::build). diff --git a/vortex-array/src/arrays/bool/mod.rs b/vortex-array/src/arrays/bool/mod.rs index 9378983286f..0a825ae72fa 100644 --- a/vortex-array/src/arrays/bool/mod.rs +++ b/vortex-array/src/arrays/bool/mod.rs @@ -4,7 +4,7 @@ mod array; mod patch; -pub use array::BoolArrayParts; +pub use array::BoolDataParts; pub use array::BoolData; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/decimal/array.rs b/vortex-array/src/arrays/decimal/array.rs index cc6b234846d..a798d5d26bc 100644 --- a/vortex-array/src/arrays/decimal/array.rs +++ b/vortex-array/src/arrays/decimal/array.rs @@ -105,7 +105,7 @@ pub struct DecimalData { pub(super) values_type: DecimalType, } -pub struct DecimalArrayParts { +pub struct DecimalDataParts { pub decimal_dtype: DecimalDType, pub values: BufferHandle, pub values_type: DecimalType, @@ -315,9 +315,9 @@ impl DecimalData { child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) } - pub fn into_parts(self) -> DecimalArrayParts { + pub fn into_parts(self) -> DecimalDataParts { let validity = self.validity(); - DecimalArrayParts { + DecimalDataParts { decimal_dtype: self.decimal_dtype, values: self.values, values_type: self.values_type, diff --git a/vortex-array/src/arrays/decimal/mod.rs b/vortex-array/src/arrays/decimal/mod.rs index 137331a492f..fa52017d829 100644 --- a/vortex-array/src/arrays/decimal/mod.rs +++ b/vortex-array/src/arrays/decimal/mod.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::DecimalArrayParts; +pub use array::DecimalDataParts; pub use array::DecimalData; pub use vtable::DecimalArray; diff --git a/vortex-array/src/arrays/dict/array.rs b/vortex-array/src/arrays/dict/array.rs index 26192b6c00c..ec0ce9e26a5 100644 --- a/vortex-array/src/arrays/dict/array.rs +++ b/vortex-array/src/arrays/dict/array.rs @@ -51,7 +51,7 @@ pub struct DictData { pub(super) all_values_referenced: bool, } -pub struct DictArrayParts { +pub struct DictDataParts { pub codes: ArrayRef, pub values: ArrayRef, } @@ -111,7 +111,7 @@ impl DictData { /// of the `values` array. Otherwise, this constructor returns an error. /// /// It is an error to provide a nullable `codes` with non-nullable `values`. - pub fn try_new(codes: ArrayRef, values: ArrayRef) -> VortexResult { + pub(crate) fn try_new(codes: ArrayRef, values: ArrayRef) -> VortexResult { if !codes.dtype().is_int() { vortex_bail!(MismatchedTypes: "int", codes.dtype()); } @@ -136,8 +136,8 @@ impl DictData { self.len() == 0 } - pub fn into_parts(mut self) -> DictArrayParts { - DictArrayParts { + pub fn into_parts(mut self) -> DictDataParts { + DictDataParts { codes: self.slots[CODES_SLOT] .take() .vortex_expect("DictArray codes slot"), diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index d8694634ce5..d817a1256bd 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -10,7 +10,7 @@ use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; -use super::DictArrayParts; +use super::DictDataParts; use super::DictData; use super::DictMetadata; use super::array::NUM_SLOTS; @@ -188,7 +188,7 @@ impl VTable for Dict { let array = require_child!(array, array.values(), 1 => AnyCanonical); - let DictArrayParts { codes, values, .. } = array.into_data().into_parts(); + let DictDataParts { codes, values, .. } = array.into_data().into_parts(); let codes = codes .try_into::() diff --git a/vortex-array/src/arrays/filter/array.rs b/vortex-array/src/arrays/filter/array.rs index e4835188d2b..6ff5adfa313 100644 --- a/vortex-array/src/arrays/filter/array.rs +++ b/vortex-array/src/arrays/filter/array.rs @@ -17,14 +17,6 @@ pub(super) const CHILD_SLOT: usize = 0; pub(super) const NUM_SLOTS: usize = 1; pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child"]; -/// Decomposed parts of the filter array. -pub struct FilterArrayParts { - /// Child array that is filtered by the mask - pub child: ArrayRef, - /// Mask to apply at filter time. Child elements with set indices are kept, the rest discarded. - pub mask: Mask, -} - // TODO(connor): Write docs on why we have this, and what we had in the old world so that the future // does not repeat the mistakes of the past. /// A lazy array that represents filtering a child array by a boolean [`Mask`]. @@ -38,12 +30,17 @@ pub struct FilterData { pub(super) mask: Mask, } +pub struct FilterDataParts { + pub child: ArrayRef, + pub mask: Mask, +} + impl FilterData { pub fn new(array: ArrayRef, mask: Mask) -> Self { Self::try_new(array, mask).vortex_expect("FilterArray construction failed") } - pub fn try_new(array: ArrayRef, mask: Mask) -> VortexResult { + fn try_new(array: ArrayRef, mask: Mask) -> VortexResult { vortex_ensure_eq!( array.len(), mask.len(), @@ -84,6 +81,15 @@ impl FilterData { pub fn filter_mask(&self) -> &Mask { &self.mask } + + pub fn into_parts(mut self) -> FilterDataParts { + FilterDataParts { + child: self.slots[CHILD_SLOT] + .take() + .vortex_expect("FilterArray child slot"), + mask: self.mask, + } + } } impl Array { @@ -104,15 +110,3 @@ impl Array { Array::try_from_parts(ArrayParts::new(Filter, dtype, len, data)) } } - -impl FilterData { - /// Consume the array and return its individual components. - pub fn into_parts(mut self) -> FilterArrayParts { - FilterArrayParts { - child: self.slots[CHILD_SLOT] - .take() - .vortex_expect("FilterArray child slot"), - mask: self.mask, - } - } -} diff --git a/vortex-array/src/arrays/filter/mod.rs b/vortex-array/src/arrays/filter/mod.rs index d10cae11e41..7491899230e 100644 --- a/vortex-array/src/arrays/filter/mod.rs +++ b/vortex-array/src/arrays/filter/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::FilterArrayParts; pub use array::FilterData; +pub use array::FilterDataParts; pub use vtable::FilterArray; mod execute; diff --git a/vortex-array/src/arrays/filter/rules.rs b/vortex-array/src/arrays/filter/rules.rs index 402a7b14585..0095d47b422 100644 --- a/vortex-array/src/arrays/filter/rules.rs +++ b/vortex-array/src/arrays/filter/rules.rs @@ -11,7 +11,7 @@ use crate::array::ArrayView; use crate::arrays::Filter; use crate::arrays::Struct; use crate::arrays::StructArray; -use crate::arrays::struct_::StructArrayParts; +use crate::arrays::struct_::StructDataParts; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ArrayReduceRule; use crate::optimizer::rules::ParentRuleSet; @@ -69,7 +69,7 @@ impl ArrayReduceRule for FilterStructRule { }; let len = mask.true_count(); - let StructArrayParts { + let StructDataParts { fields, struct_fields, validity, diff --git a/vortex-array/src/arrays/filter/vtable.rs b/vortex-array/src/arrays/filter/vtable.rs index 90deb1790e4..fb7bdbf9d0b 100644 --- a/vortex-array/src/arrays/filter/vtable.rs +++ b/vortex-array/src/arrays/filter/vtable.rs @@ -55,6 +55,28 @@ impl VTable for Filter { Self::ID } + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.child().dtype() == dtype, + "FilterArray dtype {} does not match outer dtype {}", + data.child().dtype(), + dtype + ); + vortex_ensure!( + data.len() == len, + "FilterArray length {} does not match outer length {}", + data.len(), + len + ); + vortex_ensure!( + data.child().len() == data.mask.len(), + "FilterArray child length {} does not match mask length {}", + data.child().len(), + data.mask.len() + ); + Ok(()) + } + fn array_hash(array: &FilterData, state: &mut H, precision: Precision) { array.child().array_hash(state, precision); array.mask.array_hash(state, precision); diff --git a/vortex-array/src/arrays/fixed_size_list/array.rs b/vortex-array/src/arrays/fixed_size_list/array.rs index 052f38bb127..eb186d4d583 100644 --- a/vortex-array/src/arrays/fixed_size_list/array.rs +++ b/vortex-array/src/arrays/fixed_size_list/array.rs @@ -122,7 +122,7 @@ impl FixedSizeListData { /// /// Returns an error if the provided components do not satisfy the invariants documented /// in `FixedSizeListArray::new_unchecked`. - pub fn try_new( + pub(crate) fn try_new( elements: ArrayRef, list_size: u32, validity: Validity, diff --git a/vortex-array/src/arrays/list/array.rs b/vortex-array/src/arrays/list/array.rs index 6f76fd9917c..631fe09d56e 100644 --- a/vortex-array/src/arrays/list/array.rs +++ b/vortex-array/src/arrays/list/array.rs @@ -98,11 +98,11 @@ pub struct ListData { pub(super) slots: Vec>, } -pub struct ListArrayParts { - pub dtype: DType, +pub struct ListDataParts { pub elements: ArrayRef, pub offsets: ArrayRef, pub validity: Validity, + pub dtype: DType, } impl ListData { @@ -124,7 +124,7 @@ impl ListData { /// /// Returns an error if the provided components do not satisfy the invariants documented in /// `ListArray::new_unchecked`. - pub fn try_new( + pub(crate) fn try_new( elements: ArrayRef, offsets: ArrayRef, validity: Validity, @@ -249,22 +249,6 @@ impl ListData { Ok(()) } - - /// Splits an array into its parts - pub fn into_parts(mut self) -> ListArrayParts { - let validity = self.validity(); - ListArrayParts { - dtype: self.dtype(), - elements: self.slots[ELEMENTS_SLOT] - .take() - .vortex_expect("ListArray elements slot"), - offsets: self.slots[OFFSETS_SLOT] - .take() - .vortex_expect("ListArray offsets slot"), - validity, - } - } - /// Returns the dtype of the array. pub fn dtype(&self) -> DType { DType::List(Arc::new(self.elements().dtype().clone()), self.nullability) @@ -350,6 +334,21 @@ impl ListData { .vortex_expect("ListArray elements slot") } + pub fn into_parts(mut self) -> ListDataParts { + let dtype = self.dtype(); + let validity = self.validity(); + ListDataParts { + elements: self.slots[ELEMENTS_SLOT] + .take() + .vortex_expect("ListArray elements slot"), + offsets: self.slots[OFFSETS_SLOT] + .take() + .vortex_expect("ListArray offsets slot"), + validity, + dtype, + } + } + // TODO(connor)[ListView]: Create 2 functions `reset_offsets` and `recursive_reset_offsets`, // where `reset_offsets` is infallible. // Also, `reset_offsets` can be made more efficient by replacing `sub_scalar` with a match on diff --git a/vortex-array/src/arrays/list/mod.rs b/vortex-array/src/arrays/list/mod.rs index 753a2a35758..3147097ca80 100644 --- a/vortex-array/src/arrays/list/mod.rs +++ b/vortex-array/src/arrays/list/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::ListArrayParts; pub use array::ListData; +pub use array::ListDataParts; pub use vtable::ListArray; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/listview/array.rs b/vortex-array/src/arrays/listview/array.rs index b4a3eca0181..fa7d3941657 100644 --- a/vortex-array/src/arrays/listview/array.rs +++ b/vortex-array/src/arrays/listview/array.rs @@ -125,7 +125,7 @@ pub struct ListViewData { is_zero_copy_to_list: bool, } -pub struct ListViewArrayParts { +pub struct ListViewDataParts { pub elements_dtype: Arc, /// See `ListViewArray::elements` @@ -337,9 +337,9 @@ impl ListViewData { .is_ok() } - pub fn into_parts(mut self) -> ListViewArrayParts { + pub fn into_parts(mut self) -> ListViewDataParts { let validity = self.validity(); - ListViewArrayParts { + ListViewDataParts { elements_dtype: Arc::new(self.elements().dtype().clone()), elements: self.slots[ELEMENTS_SLOT] .take() diff --git a/vortex-array/src/arrays/listview/mod.rs b/vortex-array/src/arrays/listview/mod.rs index 4ca62ef0820..aa1a1159ef5 100644 --- a/vortex-array/src/arrays/listview/mod.rs +++ b/vortex-array/src/arrays/listview/mod.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::ListViewArrayParts; +pub use array::ListViewDataParts; pub use array::ListViewData; pub use vtable::ListViewArray; diff --git a/vortex-array/src/arrays/masked/array.rs b/vortex-array/src/arrays/masked/array.rs index 1da29d7a35e..c395dedeed2 100644 --- a/vortex-array/src/arrays/masked/array.rs +++ b/vortex-array/src/arrays/masked/array.rs @@ -28,7 +28,7 @@ pub struct MaskedData { } impl MaskedData { - pub fn try_new(child: ArrayRef, validity: Validity) -> VortexResult { + pub(crate) fn try_new(child: ArrayRef, validity: Validity) -> VortexResult { if matches!(validity, Validity::NonNullable) { vortex_bail!("MaskedArray must have nullable validity, got {validity:?}") } diff --git a/vortex-array/src/arrays/patched/compute/compare.rs b/vortex-array/src/arrays/patched/compute/compare.rs index 0af82fcf7b3..54c89c1744d 100644 --- a/vortex-array/src/arrays/patched/compute/compare.rs +++ b/vortex-array/src/arrays/patched/compute/compare.rs @@ -14,7 +14,7 @@ use crate::arrays::BoolArray; use crate::arrays::ConstantArray; use crate::arrays::Patched; use crate::arrays::PrimitiveArray; -use crate::arrays::bool::BoolArrayParts; +use crate::arrays::bool::BoolDataParts; use crate::arrays::primitive::NativeValue; use crate::builtins::ArrayBuiltins; use crate::dtype::NativePType; @@ -50,12 +50,13 @@ impl CompareKernel for Patched { .execute::(ctx)? .into_bool(); - let BoolArrayParts { + let result = result.into_parts(); + let BoolDataParts { bits, offset, len, validity, - } = result.into_encoding_parts(); + } = result.data.into_parts(); let mut bits = BitBufferMut::from_buffer(bits.unwrap_host().into_mut(), offset, len); diff --git a/vortex-array/src/arrays/patched/compute/take.rs b/vortex-array/src/arrays/patched/compute/take.rs index 9ab496c953d..1257eec5169 100644 --- a/vortex-array/src/arrays/patched/compute/take.rs +++ b/vortex-array/src/arrays/patched/compute/take.rs @@ -12,7 +12,7 @@ use crate::array::ArrayView; use crate::arrays::Patched; use crate::arrays::PrimitiveArray; use crate::arrays::dict::TakeExecute; -use crate::arrays::primitive::PrimitiveArrayParts; +use crate::arrays::primitive::PrimitiveDataParts; use crate::dtype::IntegerPType; use crate::dtype::NativePType; use crate::match_each_native_ptype; @@ -35,7 +35,7 @@ impl TakeExecute for Patched { .take(indices.clone())? .execute::(ctx)?; - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer, validity, ptype, diff --git a/vortex-array/src/arrays/patched/vtable/mod.rs b/vortex-array/src/arrays/patched/vtable/mod.rs index f319ccc85e2..c772e94bb38 100644 --- a/vortex-array/src/arrays/patched/vtable/mod.rs +++ b/vortex-array/src/arrays/patched/vtable/mod.rs @@ -38,7 +38,7 @@ use crate::arrays::patched::array::NUM_SLOTS; use crate::arrays::patched::array::SLOT_NAMES; use crate::arrays::patched::compute::rules::PARENT_RULES; use crate::arrays::patched::vtable::kernels::PARENT_KERNELS; -use crate::arrays::primitive::PrimitiveArrayParts; +use crate::arrays::primitive::PrimitiveDataParts; use crate::buffer::BufferHandle; use crate::builders::ArrayBuilder; use crate::builders::PrimitiveBuilder; @@ -266,7 +266,7 @@ impl VTable for Patched { .execute::(ctx)? .into_primitive(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer, ptype, validity, diff --git a/vortex-array/src/arrays/primitive/array/mod.rs b/vortex-array/src/arrays/primitive/array/mod.rs index b58cb7e68a0..0675c50d9f1 100644 --- a/vortex-array/src/arrays/primitive/array/mod.rs +++ b/vortex-array/src/arrays/primitive/array/mod.rs @@ -81,7 +81,7 @@ pub struct PrimitiveData { pub(super) buffer: BufferHandle, } -pub struct PrimitiveArrayParts { +pub struct PrimitiveDataParts { pub ptype: PType, pub buffer: BufferHandle, pub validity: Validity, @@ -295,10 +295,10 @@ impl Array { impl PrimitiveData { /// Consume the primitive array and returns its component parts. - pub fn into_parts(self) -> PrimitiveArrayParts { + pub fn into_parts(self) -> PrimitiveDataParts { let ptype = self.ptype(); let validity = self.validity(); - PrimitiveArrayParts { + PrimitiveDataParts { ptype, buffer: self.buffer, validity, diff --git a/vortex-array/src/arrays/primitive/mod.rs b/vortex-array/src/arrays/primitive/mod.rs index 522a03afc86..e247e22c6d4 100644 --- a/vortex-array/src/arrays/primitive/mod.rs +++ b/vortex-array/src/arrays/primitive/mod.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::PrimitiveArrayParts; +pub use array::PrimitiveDataParts; pub use array::PrimitiveData; pub use array::chunk_range; pub use array::patch_chunk; diff --git a/vortex-array/src/arrays/slice/array.rs b/vortex-array/src/arrays/slice/array.rs index 6c2f38e9832..1292bcfbc01 100644 --- a/vortex-array/src/arrays/slice/array.rs +++ b/vortex-array/src/arrays/slice/array.rs @@ -24,13 +24,13 @@ pub struct SliceData { pub(super) range: Range, } -pub struct SliceArrayParts { +pub struct SliceDataParts { pub child: ArrayRef, pub range: Range, } impl SliceData { - pub fn try_new(child: ArrayRef, range: Range) -> VortexResult { + fn try_new(child: ArrayRef, range: Range) -> VortexResult { if range.end > child.len() { vortex_panic!( "SliceArray range out of bounds: range {:?} exceeds child array length {}", @@ -74,6 +74,15 @@ impl SliceData { .as_ref() .vortex_expect("SliceArray child slot") } + + pub fn into_parts(mut self) -> SliceDataParts { + SliceDataParts { + child: self.slots[CHILD_SLOT] + .take() + .vortex_expect("SliceArray child slot"), + range: self.range, + } + } } impl Array { @@ -94,15 +103,3 @@ impl Array { .vortex_expect("SliceData is always valid") } } - -impl SliceData { - /// Consume the slice array and return its components. - pub fn into_parts(mut self) -> SliceArrayParts { - SliceArrayParts { - child: self.slots[CHILD_SLOT] - .take() - .vortex_expect("SliceArray child slot"), - range: self.range, - } - } -} diff --git a/vortex-array/src/arrays/slice/mod.rs b/vortex-array/src/arrays/slice/mod.rs index 928367ea213..f701a274a14 100644 --- a/vortex-array/src/arrays/slice/mod.rs +++ b/vortex-array/src/arrays/slice/mod.rs @@ -15,8 +15,8 @@ mod vtable; use std::ops::Range; -pub use array::SliceArrayParts; pub use array::SliceData; +pub use array::SliceDataParts; use vortex_error::VortexResult; pub use vtable::*; diff --git a/vortex-array/src/arrays/slice/vtable.rs b/vortex-array/src/arrays/slice/vtable.rs index 5e0b52c91fd..562fcd4c117 100644 --- a/vortex-array/src/arrays/slice/vtable.rs +++ b/vortex-array/src/arrays/slice/vtable.rs @@ -57,6 +57,28 @@ impl VTable for Slice { Slice::ID } + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.child().dtype() == dtype, + "SliceArray dtype {} does not match outer dtype {}", + data.child().dtype(), + dtype + ); + vortex_ensure!( + data.len() == len, + "SliceArray length {} does not match outer length {}", + data.len(), + len + ); + vortex_ensure!( + data.range.end <= data.child().len(), + "SliceArray range {:?} exceeds child length {}", + data.range, + data.child().len() + ); + Ok(()) + } + fn array_hash(array: &SliceData, state: &mut H, precision: Precision) { array.child().array_hash(state, precision); array.range.start.hash(state); diff --git a/vortex-array/src/arrays/struct_/array.rs b/vortex-array/src/arrays/struct_/array.rs index 1ec06c97822..032fc426f58 100644 --- a/vortex-array/src/arrays/struct_/array.rs +++ b/vortex-array/src/arrays/struct_/array.rs @@ -155,7 +155,7 @@ pub struct StructData { pub(super) slots: Vec>, } -pub struct StructArrayParts { +pub struct StructDataParts { pub struct_fields: StructFields, pub fields: Arc<[ArrayRef]>, pub validity: Validity, @@ -273,7 +273,7 @@ impl StructData { /// /// Returns an error if the provided components do not satisfy the invariants documented in /// `StructArray::new_unchecked`. - pub fn try_new( + pub(crate) fn try_new( names: FieldNames, fields: impl Into>, length: usize, @@ -392,7 +392,7 @@ impl StructData { Ok(()) } - pub fn try_new_with_dtype( + pub(crate) fn try_new_with_dtype( fields: impl Into>, dtype: StructFields, length: usize, @@ -405,7 +405,7 @@ impl StructData { Ok(unsafe { Self::new_unchecked(fields, dtype, length, validity) }) } - pub fn into_parts(self) -> StructArrayParts { + pub fn into_parts(self) -> StructDataParts { let StructData { names, slots, @@ -431,7 +431,7 @@ impl StructData { .skip(FIELDS_OFFSET) .map(|s| s.vortex_expect("StructArray field slot")) .collect(); - StructArrayParts { + StructDataParts { struct_fields, fields, validity, @@ -638,11 +638,6 @@ impl Array { Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) } - /// Decompose this struct array into its encoding-specific constituent parts. - pub fn into_encoding_parts(self) -> StructArrayParts { - self.into_data().into_parts() - } - /// Create a `StructArray` from an iterator of (name, array) pairs with validity. pub fn try_from_iter_with_validity< N: AsRef, diff --git a/vortex-array/src/arrays/struct_/mod.rs b/vortex-array/src/arrays/struct_/mod.rs index b867f903901..2a9d6e1df07 100644 --- a/vortex-array/src/arrays/struct_/mod.rs +++ b/vortex-array/src/arrays/struct_/mod.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::StructArrayParts; +pub use array::StructDataParts; pub use array::StructData; pub use vtable::StructArray; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/varbinview/array.rs b/vortex-array/src/arrays/varbinview/array.rs index ab06e8a9229..52e58c72572 100644 --- a/vortex-array/src/arrays/varbinview/array.rs +++ b/vortex-array/src/arrays/varbinview/array.rs @@ -102,7 +102,7 @@ pub struct VarBinViewData { pub(super) views: BufferHandle, } -pub struct VarBinViewArrayParts { +pub struct VarBinViewDataParts { pub dtype: DType, pub buffers: Arc<[BufferHandle]>, pub views: BufferHandle, @@ -406,9 +406,9 @@ impl VarBinViewData { } /// Splits the array into owned parts - pub fn into_parts(self) -> VarBinViewArrayParts { + pub fn into_parts(self) -> VarBinViewDataParts { let validity = self.validity(); - VarBinViewArrayParts { + VarBinViewDataParts { dtype: self.dtype(), buffers: self.buffers, views: self.views, diff --git a/vortex-array/src/arrays/varbinview/mod.rs b/vortex-array/src/arrays/varbinview/mod.rs index d29fd5bf3e5..1a56329cda3 100644 --- a/vortex-array/src/arrays/varbinview/mod.rs +++ b/vortex-array/src/arrays/varbinview/mod.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::VarBinViewArrayParts; +pub use array::VarBinViewDataParts; pub use array::VarBinViewData; pub use vtable::VarBinViewArray; diff --git a/vortex-array/src/arrays/variant/mod.rs b/vortex-array/src/arrays/variant/mod.rs index 5ae1f503820..9cf59ac5197 100644 --- a/vortex-array/src/arrays/variant/mod.rs +++ b/vortex-array/src/arrays/variant/mod.rs @@ -21,8 +21,8 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child"]; /// (e.g. a `ParquetVariantArray` or any other variant encoding). /// /// Nullability is delegated to the child array: `VariantArray`'s dtype is -/// always the child's dtype. The child's validity determines which rows are -/// null. +/// always `DType::Variant(child.dtype().nullability())`. The child's validity +/// determines which rows are null. #[derive(Clone, Debug)] pub struct VariantData { pub(super) slots: Vec>, @@ -42,8 +42,8 @@ impl VariantData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - self.child().dtype() + pub fn dtype(&self) -> DType { + DType::Variant(self.child().dtype().nullability()) } /// Returns `true` if this array is empty. @@ -62,12 +62,11 @@ impl VariantData { impl Array { /// Creates a new `VariantArray`. pub fn new(child: ArrayRef) -> Self { - let dtype = child.dtype().clone(); + let dtype = DType::Variant(child.dtype().nullability()); let len = child.len(); - let stats = child.statistics().to_array_stats(); - Array::try_from_parts( - ArrayParts::new(Variant, dtype, len, VariantData::new(child)).with_stats(stats), - ) - .vortex_expect("VariantData is always valid") + let stats = child.statistics().to_owned(); + Array::try_from_parts(ArrayParts::new(Variant, dtype, len, VariantData::new(child))) + .map(|array| array.with_stats_set(stats)) + .vortex_expect("VariantData is always valid") } } diff --git a/vortex-array/src/arrays/variant/vtable/mod.rs b/vortex-array/src/arrays/variant/vtable/mod.rs index 7b36379a76a..497a889fbc2 100644 --- a/vortex-array/src/arrays/variant/vtable/mod.rs +++ b/vortex-array/src/arrays/variant/vtable/mod.rs @@ -49,6 +49,26 @@ impl VTable for Variant { Self::ID } + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + matches!(dtype, DType::Variant(_)), + "Expected Variant DType, got {dtype}" + ); + vortex_ensure!( + data.child().dtype() == dtype, + "VariantArray child dtype {} does not match outer dtype {}", + data.child().dtype(), + dtype + ); + vortex_ensure!( + data.len() == len, + "VariantArray length {} does not match outer length {}", + data.len(), + len + ); + Ok(()) + } + fn array_hash(array: &VariantData, state: &mut H, precision: Precision) { array.child().array_hash(state, precision); } @@ -130,11 +150,16 @@ impl VTable for Variant { mod tests { use super::*; use crate::IntoArray; - use crate::arrays::PrimitiveArray; + use crate::arrays::ConstantArray; + use crate::dtype::DType; + use crate::dtype::Nullability; + use crate::scalar::Scalar; #[test] fn with_slots_rejects_missing_child() { - let array = VariantArray::new(PrimitiveArray::from_iter([1u8, 2, 3]).into_array()); + let child = + ConstantArray::new(Scalar::null(DType::Variant(Nullability::Nullable)), 3).into_array(); + let array = VariantArray::new(child); let mut data = array.into_data(); let err = ::with_slots(&mut data, vec![None]).unwrap_err(); diff --git a/vortex-array/src/arrow/executor/dictionary.rs b/vortex-array/src/arrow/executor/dictionary.rs index 5d9b75182b3..23cc71e65dd 100644 --- a/vortex-array/src/arrow/executor/dictionary.rs +++ b/vortex-array/src/arrow/executor/dictionary.rs @@ -21,7 +21,7 @@ use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::Dict; use crate::arrays::DictArray; -use crate::arrays::dict::DictArrayParts; +use crate::arrays::dict::DictDataParts; use crate::arrow::ArrowArrayExecutor; pub(super) fn to_arrow_dictionary( @@ -78,7 +78,7 @@ fn dict_to_dict( values_type: &DataType, ctx: &mut ExecutionCtx, ) -> VortexResult { - let DictArrayParts { codes, values, .. } = array.into_data().into_parts(); + let DictDataParts { codes, values, .. } = array.into_data().into_parts(); let codes = codes.execute_arrow(Some(codes_type), ctx)?; let values = values.execute_arrow(Some(values_type), ctx)?; make_dict_array(codes_type, codes, values) diff --git a/vortex-array/src/arrow/executor/list.rs b/vortex-array/src/arrow/executor/list.rs index 661e459f62e..f17ededa54d 100644 --- a/vortex-array/src/arrow/executor/list.rs +++ b/vortex-array/src/arrow/executor/list.rs @@ -21,7 +21,7 @@ use crate::arrays::List; use crate::arrays::ListArray; use crate::arrays::ListView; use crate::arrays::ListViewArray; -use crate::arrays::listview::ListViewArrayParts; +use crate::arrays::listview::ListViewDataParts; use crate::arrays::listview::ListViewRebuildMode; use crate::arrow::ArrowArrayExecutor; use crate::arrow::executor::validity::to_arrow_null_buffer; @@ -133,7 +133,7 @@ fn list_view_zctl( ))); } - let ListViewArrayParts { + let ListViewDataParts { elements, offsets, sizes, diff --git a/vortex-array/src/arrow/executor/list_view.rs b/vortex-array/src/arrow/executor/list_view.rs index fb536bc3c1b..614ea6e7c93 100644 --- a/vortex-array/src/arrow/executor/list_view.rs +++ b/vortex-array/src/arrow/executor/list_view.rs @@ -14,7 +14,7 @@ use crate::ExecutionCtx; use crate::arrays::ListView; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; -use crate::arrays::listview::ListViewArrayParts; +use crate::arrays::listview::ListViewDataParts; use crate::arrow::ArrowArrayExecutor; use crate::arrow::executor::validity::to_arrow_null_buffer; use crate::builtins::ArrayBuiltins; @@ -43,7 +43,7 @@ fn list_view_to_list_view( elements_field: &FieldRef, ctx: &mut ExecutionCtx, ) -> VortexResult { - let ListViewArrayParts { + let ListViewDataParts { elements, offsets, sizes, diff --git a/vortex-array/src/arrow/executor/struct_.rs b/vortex-array/src/arrow/executor/struct_.rs index 02deda5913d..7edd1b05e35 100644 --- a/vortex-array/src/arrow/executor/struct_.rs +++ b/vortex-array/src/arrow/executor/struct_.rs @@ -19,7 +19,7 @@ use crate::arrays::Chunked; use crate::arrays::ScalarFnVTable; use crate::arrays::Struct; use crate::arrays::StructArray; -use crate::arrays::struct_::StructArrayParts; +use crate::arrays::struct_::StructDataParts; use crate::arrow::ArrowArrayExecutor; use crate::arrow::executor::validity::to_arrow_null_buffer; use crate::builtins::ArrayBuiltins; @@ -49,13 +49,14 @@ pub(super) fn to_arrow_struct( // Attempt to short-circuit if the array is already a Struct: let array = match array.try_into::() { Ok(array) => { - let len = array.len(); - let StructArrayParts { + let parts = array.into_parts(); + let len = parts.len; + let StructDataParts { validity, fields, struct_fields, .. - } = array.into_encoding_parts(); + } = parts.data.into_parts(); let validity = to_arrow_null_buffer(validity, len, ctx)?; return create_from_fields( target_fields.ok_or_else(|| struct_fields.names().clone()), @@ -97,13 +98,14 @@ pub(super) fn to_arrow_struct( }; let struct_array = array.execute::(ctx)?; - let len = struct_array.len(); - let StructArrayParts { + let parts = struct_array.into_parts(); + let len = parts.len; + let StructDataParts { validity, fields, struct_fields, .. - } = struct_array.into_encoding_parts(); + } = parts.data.into_parts(); let validity = to_arrow_null_buffer(validity, len, ctx)?; create_from_fields( diff --git a/vortex-array/src/canonical.rs b/vortex-array/src/canonical.rs index c4714b07ed1..e50458bcebc 100644 --- a/vortex-array/src/canonical.rs +++ b/vortex-array/src/canonical.rs @@ -37,13 +37,13 @@ use crate::arrays::VarBinView; use crate::arrays::VarBinViewArray; use crate::arrays::Variant; use crate::arrays::VariantArray; -use crate::arrays::bool::BoolArrayParts; -use crate::arrays::decimal::DecimalArrayParts; -use crate::arrays::listview::ListViewArrayParts; +use crate::arrays::bool::BoolDataParts; +use crate::arrays::decimal::DecimalDataParts; +use crate::arrays::listview::ListViewDataParts; use crate::arrays::listview::ListViewRebuildMode; -use crate::arrays::primitive::PrimitiveArrayParts; -use crate::arrays::struct_::StructArrayParts; -use crate::arrays::varbinview::VarBinViewArrayParts; +use crate::arrays::primitive::PrimitiveDataParts; +use crate::arrays::struct_::StructDataParts; +use crate::arrays::varbinview::VarBinViewDataParts; use crate::dtype::DType; use crate::dtype::NativePType; use crate::dtype::Nullability; @@ -544,18 +544,19 @@ impl Executable for CanonicalValidity { match array.execute::(ctx)? { n @ Canonical::Null(_) => Ok(CanonicalValidity(n)), Canonical::Bool(b) => { - let BoolArrayParts { + let b = b.into_parts(); + let BoolDataParts { bits, offset, len, validity, - } = b.into_encoding_parts(); + } = b.data.into_parts(); Ok(CanonicalValidity(Canonical::Bool( BoolArray::try_new_from_handle(bits, offset, len, validity.execute(ctx)?)?, ))) } Canonical::Primitive(p) => { - let PrimitiveArrayParts { + let PrimitiveDataParts { ptype, buffer, validity, @@ -565,7 +566,7 @@ impl Executable for CanonicalValidity { }))) } Canonical::Decimal(d) => { - let DecimalArrayParts { + let DecimalDataParts { decimal_dtype, values, values_type, @@ -581,7 +582,7 @@ impl Executable for CanonicalValidity { }))) } Canonical::VarBinView(vbv) => { - let VarBinViewArrayParts { + let VarBinViewDataParts { dtype, buffers, views, @@ -598,7 +599,7 @@ impl Executable for CanonicalValidity { } Canonical::List(l) => { let zctl = l.is_zero_copy_to_list(); - let ListViewArrayParts { + let ListViewDataParts { elements, offsets, sizes, @@ -619,12 +620,13 @@ impl Executable for CanonicalValidity { ))) } Canonical::Struct(st) => { - let len = st.len(); - let StructArrayParts { + let parts = st.into_parts(); + let len = parts.len; + let StructDataParts { struct_fields, fields, validity, - } = st.into_encoding_parts(); + } = parts.data.into_parts(); Ok(CanonicalValidity(Canonical::Struct(unsafe { StructArray::new_unchecked(fields, struct_fields, len, validity.execute(ctx)?) }))) @@ -664,18 +666,19 @@ impl Executable for RecursiveCanonical { match array.execute::(ctx)? { n @ Canonical::Null(_) => Ok(RecursiveCanonical(n)), Canonical::Bool(b) => { - let BoolArrayParts { + let b = b.into_parts(); + let BoolDataParts { bits, offset, len, validity, - } = b.into_encoding_parts(); + } = b.data.into_parts(); Ok(RecursiveCanonical(Canonical::Bool( BoolArray::try_new_from_handle(bits, offset, len, validity.execute(ctx)?)?, ))) } Canonical::Primitive(p) => { - let PrimitiveArrayParts { + let PrimitiveDataParts { ptype, buffer, validity, @@ -685,7 +688,7 @@ impl Executable for RecursiveCanonical { }))) } Canonical::Decimal(d) => { - let DecimalArrayParts { + let DecimalDataParts { decimal_dtype, values, values_type, @@ -701,7 +704,7 @@ impl Executable for RecursiveCanonical { }))) } Canonical::VarBinView(vbv) => { - let VarBinViewArrayParts { + let VarBinViewDataParts { dtype, buffers, views, @@ -718,7 +721,7 @@ impl Executable for RecursiveCanonical { } Canonical::List(l) => { let zctl = l.is_zero_copy_to_list(); - let ListViewArrayParts { + let ListViewDataParts { elements, offsets, sizes, @@ -749,12 +752,13 @@ impl Executable for RecursiveCanonical { ))) } Canonical::Struct(st) => { - let len = st.len(); - let StructArrayParts { + let parts = st.into_parts(); + let len = parts.len; + let StructDataParts { struct_fields, fields, validity, - } = st.into_encoding_parts(); + } = parts.data.into_parts(); let executed_fields = fields .iter() .map(|f| Ok(f.clone().execute::(ctx)?.0.into_array())) diff --git a/vortex-array/src/display/extractors/metadata.rs b/vortex-array/src/display/extractors/metadata.rs index 3bbe9c995e0..f1776f3d383 100644 --- a/vortex-array/src/display/extractors/metadata.rs +++ b/vortex-array/src/display/extractors/metadata.rs @@ -2,8 +2,6 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::fmt; -use std::fmt::Debug; - use crate::ArrayRef; use crate::display::extractor::IndentedFormatter; use crate::display::extractor::TreeContext; @@ -21,12 +19,7 @@ impl TreeExtractor for MetadataExtractor { ) -> fmt::Result { let (indent, f) = f.parts(); write!(f, "{indent}metadata: ")?; - match array.metadata() { - Ok(Some(metadata)) if metadata.is_empty() => write!(f, "EmptyMetadata")?, - Ok(Some(metadata)) => Debug::fmt(&metadata, f)?, - Ok(None) => write!(f, "")?, - Err(err) => write!(f, "")?, - } + array.metadata_fmt(f)?; writeln!(f) } } diff --git a/vortex-array/src/stats/array.rs b/vortex-array/src/stats/array.rs index b3838bd4f9e..bd2e3cf7dea 100644 --- a/vortex-array/src/stats/array.rs +++ b/vortex-array/src/stats/array.rs @@ -83,6 +83,10 @@ impl From for StatsSet { } impl StatsSetRef<'_> { + pub(crate) fn replace(&self, stats: StatsSet) { + *self.array_stats.inner.write() = stats; + } + pub fn set_iter(&self, iter: StatsSetIntoIter) { let mut guard = self.array_stats.inner.write(); for (stat, value) in iter { diff --git a/vortex-cuda/benches/zstd_cuda.rs b/vortex-cuda/benches/zstd_cuda.rs index 7fe68380439..d3e49d03c48 100644 --- a/vortex-cuda/benches/zstd_cuda.rs +++ b/vortex-cuda/benches/zstd_cuda.rs @@ -15,7 +15,7 @@ use futures::executor::block_on; use vortex::array::arrays::VarBinViewArray; use vortex::encodings::zstd::Zstd; use vortex::encodings::zstd::ZstdArray; -use vortex::encodings::zstd::ZstdArrayParts; +use vortex::encodings::zstd::ZstdDataParts; use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::error::vortex_err; @@ -138,7 +138,7 @@ fn benchmark_zstd_cuda_decompress(c: &mut Criterion) { let mut total_time = Duration::ZERO; for _ in 0..iters { - let ZstdArrayParts { + let ZstdDataParts { frames, metadata, .. } = zstd_array.clone().into_data().into_parts(); let exec = block_on(zstd_kernel_prepare(frames, &metadata, &mut cuda_ctx)) diff --git a/vortex-cuda/src/arrow/canonical.rs b/vortex-cuda/src/arrow/canonical.rs index b3bb2824b4c..605d2a9e356 100644 --- a/vortex-cuda/src/arrow/canonical.rs +++ b/vortex-cuda/src/arrow/canonical.rs @@ -7,10 +7,10 @@ use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::ToCanonical; use vortex::array::arrays::StructArray; -use vortex::array::arrays::bool::BoolArrayParts; -use vortex::array::arrays::decimal::DecimalArrayParts; -use vortex::array::arrays::primitive::PrimitiveArrayParts; -use vortex::array::arrays::struct_::StructArrayParts; +use vortex::array::arrays::bool::BoolDataParts; +use vortex::array::arrays::decimal::DecimalDataParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; +use vortex::array::arrays::struct_::StructDataParts; use vortex::array::buffer::BufferHandle; use vortex::dtype::DecimalType; use vortex::error::VortexResult; @@ -66,7 +66,7 @@ fn export_canonical( Canonical::Struct(struct_array) => export_struct(struct_array, ctx).await, Canonical::Primitive(primitive) => { let len = primitive.len(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer, validity, .. } = primitive.into_data().into_parts(); @@ -90,7 +90,7 @@ fn export_canonical( } Canonical::Decimal(decimal) => { let len = decimal.len(); - let DecimalArrayParts { + let DecimalDataParts { values, values_type, validity, @@ -118,7 +118,7 @@ fn export_canonical( let values = extension.storage_array().to_primitive(); let len = extension.len(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer, validity, .. } = values.into_data().into_parts(); @@ -128,13 +128,13 @@ fn export_canonical( export_fixed_size(buffer, len, 0, ctx) } Canonical::Bool(bool_array) => { - let BoolArrayParts { + let BoolDataParts { bits, offset, len, validity, .. - } = bool_array.into_parts(); + } = bool_array.into_data().into_parts(); check_validity_empty(&validity)?; @@ -182,9 +182,9 @@ async fn export_struct( ctx: &mut CudaExecutionCtx, ) -> VortexResult<(ArrowArray, SyncEvent)> { let len = array.len(); - let StructArrayParts { + let StructDataParts { validity, fields, .. - } = array.into_parts(); + } = array.into_data().into_parts(); check_validity_empty(&validity)?; diff --git a/vortex-cuda/src/arrow/varbinview.rs b/vortex-cuda/src/arrow/varbinview.rs index 12edc39c9bf..72ea96437dd 100644 --- a/vortex-cuda/src/arrow/varbinview.rs +++ b/vortex-cuda/src/arrow/varbinview.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use cudarc::driver::LaunchConfig; use cudarc::driver::PushKernelArg; use vortex::array::arrays::VarBinViewArray; -use vortex::array::arrays::varbinview::VarBinViewArrayParts; +use vortex::array::arrays::varbinview::VarBinViewDataParts; use vortex::array::buffer::BufferHandle; use vortex::error::VortexExpect; use vortex::error::VortexResult; @@ -33,7 +33,7 @@ pub(crate) async fn copy_varbinview_to_varbin( ctx: &mut CudaExecutionCtx, ) -> VortexResult { let len = array.len(); - let VarBinViewArrayParts { + let VarBinViewDataParts { views, buffers, validity, diff --git a/vortex-cuda/src/canonical.rs b/vortex-cuda/src/canonical.rs index 0cbfbf718cc..da87deb9f89 100644 --- a/vortex-cuda/src/canonical.rs +++ b/vortex-cuda/src/canonical.rs @@ -13,12 +13,12 @@ use vortex::array::arrays::ExtensionArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; use vortex::array::arrays::VarBinViewArray; -use vortex::array::arrays::bool::BoolArrayParts; -use vortex::array::arrays::decimal::DecimalArrayParts; -use vortex::array::arrays::primitive::PrimitiveArrayParts; -use vortex::array::arrays::struct_::StructArrayParts; +use vortex::array::arrays::bool::BoolDataParts; +use vortex::array::arrays::decimal::DecimalDataParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; +use vortex::array::arrays::struct_::StructDataParts; use vortex::array::arrays::varbinview::BinaryView; -use vortex::array::arrays::varbinview::VarBinViewArrayParts; +use vortex::array::arrays::varbinview::VarBinViewDataParts; use vortex::array::buffer::BufferHandle; use vortex::buffer::BitBuffer; use vortex::buffer::Buffer; @@ -40,12 +40,12 @@ impl CanonicalCudaExt for Canonical { Canonical::Struct(struct_array) => { // Children should all be canonical now let len = struct_array.len(); - let StructArrayParts { + let StructDataParts { fields, struct_fields, validity, .. - } = struct_array.into_parts(); + } = struct_array.into_data().into_parts(); let mut host_fields = vec![]; for field in fields.iter() { @@ -63,19 +63,19 @@ impl CanonicalCudaExt for Canonical { Canonical::Bool(bool) => { // NOTE: update to copy to host when adding buffer handle. // Also update other method to copy validity to host. - let BoolArrayParts { + let BoolDataParts { bits, validity, offset, len, .. - } = bool.into_parts(); + } = bool.into_data().into_parts(); let bits = BitBuffer::new_with_offset(bits.try_into_host()?.await?, offset, len); Ok(Canonical::Bool(BoolArray::new(bits, validity))) } Canonical::Primitive(prim) => { - let PrimitiveArrayParts { + let PrimitiveDataParts { ptype, buffer, validity, @@ -88,7 +88,7 @@ impl CanonicalCudaExt for Canonical { ))) } Canonical::Decimal(decimal) => { - let DecimalArrayParts { + let DecimalDataParts { decimal_dtype, values, values_type, @@ -105,7 +105,7 @@ impl CanonicalCudaExt for Canonical { })) } Canonical::VarBinView(varbinview) => { - let VarBinViewArrayParts { + let VarBinViewDataParts { views, buffers, validity, diff --git a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs index a59a69a173d..19e19119eb0 100644 --- a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs +++ b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs @@ -51,7 +51,7 @@ fn is_dyn_dispatch_compatible(array: &ArrayRef) -> bool { return arr.patches().is_none() && arr.dtype().as_ptype() == PType::F32; } if id == BitPacked::ID { - return array.as_::().patches().is_none(); + return array.as_::().patches(array.len()).is_none(); } if id == Dict::ID { let arr = array.as_::(); @@ -412,7 +412,7 @@ impl FusedPlan { fn walk_bitpacked(&mut self, array: ArrayRef) -> VortexResult { let bp = array.as_::(); - if bp.patches().is_some() { + if bp.patches(array.len()).is_some() { vortex_bail!("Dynamic dispatch does not support BitPackedArray with patches"); } diff --git a/vortex-cuda/src/executor.rs b/vortex-cuda/src/executor.rs index 9ae518540bd..4a14be98968 100644 --- a/vortex-cuda/src/executor.rs +++ b/vortex-cuda/src/executor.rs @@ -21,7 +21,7 @@ use vortex::array::ExecutionCtx; use vortex::array::IntoArray; use vortex::array::arrays::Struct; use vortex::array::arrays::StructArray; -use vortex::array::arrays::struct_::StructArrayParts; +use vortex::array::arrays::struct_::StructDataParts; use vortex::array::buffer::BufferHandle; use vortex::dtype::PType; use vortex::error::VortexResult; @@ -360,12 +360,12 @@ impl CudaArrayExt for ArrayRef { async fn execute_cuda(self, ctx: &mut CudaExecutionCtx) -> VortexResult { if self.encoding_id() == Struct::ID { let len = self.len(); - let StructArrayParts { + let StructDataParts { fields, struct_fields, validity, .. - } = self.try_into::().unwrap().into_parts(); + } = self.try_into::().unwrap().into_data().into_parts(); let mut cuda_fields = Vec::with_capacity(fields.len()); for field in fields.iter() { diff --git a/vortex-cuda/src/kernel/arrays/dict.rs b/vortex-cuda/src/kernel/arrays/dict.rs index 4015bbcddbb..de06c5fa234 100644 --- a/vortex-cuda/src/kernel/arrays/dict.rs +++ b/vortex-cuda/src/kernel/arrays/dict.rs @@ -15,10 +15,10 @@ use vortex::array::arrays::Dict; use vortex::array::arrays::DictArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::VarBinViewArray; -use vortex::array::arrays::decimal::DecimalArrayParts; -use vortex::array::arrays::dict::DictArrayParts; -use vortex::array::arrays::primitive::PrimitiveArrayParts; -use vortex::array::arrays::varbinview::VarBinViewArrayParts; +use vortex::array::arrays::decimal::DecimalDataParts; +use vortex::array::arrays::dict::DictDataParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; +use vortex::array::arrays::varbinview::VarBinViewDataParts; use vortex::array::buffer::BufferHandle; use vortex::array::match_each_decimal_value_type; use vortex::array::match_each_integer_ptype; @@ -65,7 +65,7 @@ impl CudaExecute for DictExecutor { #[expect(clippy::cognitive_complexity)] async fn execute_dict_prim(dict: DictArray, ctx: &mut CudaExecutionCtx) -> VortexResult { - let DictArrayParts { values, codes, .. } = dict.into_data().into_parts(); + let DictDataParts { values, codes, .. } = dict.into_data().into_parts(); // Execute both children to get them as primitives on the device let values_canonical = values.execute_cuda(ctx).await?; @@ -93,14 +93,14 @@ async fn execute_dict_prim_typed VortexResult { - let DictArrayParts { + let dtype = dict.dtype().clone(); + let DictDataParts { values, codes, - dtype, .. } = dict.into_data().into_parts(); @@ -179,14 +179,14 @@ async fn execute_dict_decimal_typed< let codes_len = codes.len(); let codes_len_u64 = codes_len as u64; - let DecimalArrayParts { + let DecimalDataParts { values: values_buffer, validity: values_validity, .. } = values.into_data().into_parts(); let output_validity = values_validity.take(&codes.clone().into_array())?; - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer: codes_buffer, .. } = codes.into_data().into_parts(); @@ -234,10 +234,10 @@ async fn execute_dict_varbinview( dict: DictArray, ctx: &mut CudaExecutionCtx, ) -> VortexResult { - let DictArrayParts { + let dtype = dict.dtype().clone(); + let DictDataParts { values, codes, - dtype, .. } = dict.into_data().into_parts(); @@ -246,7 +246,7 @@ async fn execute_dict_varbinview( let codes_len = codes_prim.len(); let values_vbv = values.execute_cuda(ctx).await?.into_varbinview(); - let VarBinViewArrayParts { + let VarBinViewDataParts { views: values_views_handle, buffers: values_data_buffers, validity: values_validity, @@ -254,7 +254,7 @@ async fn execute_dict_varbinview( } = values_vbv.into_data().into_parts(); let output_validity = values_validity.take(&codes_prim.clone().into_array())?; - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer: codes_buffer, .. } = codes_prim.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/encodings/alp.rs b/vortex-cuda/src/kernel/encodings/alp.rs index e33a35bbf56..1635927a42e 100644 --- a/vortex-cuda/src/kernel/encodings/alp.rs +++ b/vortex-cuda/src/kernel/encodings/alp.rs @@ -11,7 +11,7 @@ use tracing::instrument; use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::arrays::PrimitiveArray; -use vortex::array::arrays::primitive::PrimitiveArrayParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::array::buffer::BufferHandle; use vortex::array::match_each_unsigned_integer_ptype; use vortex::dtype::NativePType; @@ -46,7 +46,9 @@ impl CudaExecute for ALPExecutor { .try_into::() .map_err(|_| vortex_err!("Expected ALPArray"))?; - match_each_alp_float_ptype!(array.ptype(), |A| { decode_alp::(array, ctx).await }) + match_each_alp_float_ptype!(array.dtype().as_ptype(), |A| { + decode_alp::(array, ctx).await + }) } } @@ -66,7 +68,7 @@ where // Execute child and copy to device let canonical = array.encoded().clone().execute_cuda(ctx).await?; let primitive = canonical.into_primitive(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer, validity, .. } = primitive.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/encodings/bitpacked.rs b/vortex-cuda/src/kernel/encodings/bitpacked.rs index 4ef8b9bf1d8..7ed4f658610 100644 --- a/vortex-cuda/src/kernel/encodings/bitpacked.rs +++ b/vortex-cuda/src/kernel/encodings/bitpacked.rs @@ -18,7 +18,7 @@ use vortex::array::match_each_integer_ptype; use vortex::dtype::NativePType; use vortex::encodings::fastlanes::BitPacked; use vortex::encodings::fastlanes::BitPackedArray; -use vortex::encodings::fastlanes::BitPackedArrayParts; +use vortex::encodings::fastlanes::BitPackedDataParts; use vortex::encodings::fastlanes::unpack_iter::BitPacked as BitPackedUnpack; use vortex::error::VortexResult; use vortex::error::vortex_ensure; @@ -52,7 +52,7 @@ impl CudaExecute for BitPackedExecutor { let array = Self::try_specialize(array).ok_or_else(|| vortex_err!("Expected BitPackedArray"))?; - match_each_integer_ptype!(array.ptype(), |A| { + match_each_integer_ptype!(array.ptype(array.dtype()), |A| { decode_bitpacked::(array, A::default(), ctx).await }) } @@ -96,7 +96,7 @@ where A: BitPackedUnpack + NativePType + DeviceRepr + Send + Sync + 'static, A::Physical: DeviceRepr + Send + Sync + 'static, { - let BitPackedArrayParts { + let BitPackedDataParts { offset, bit_width, len, diff --git a/vortex-cuda/src/kernel/encodings/date_time_parts.rs b/vortex-cuda/src/kernel/encodings/date_time_parts.rs index da21f267d32..a0c4a446567 100644 --- a/vortex-cuda/src/kernel/encodings/date_time_parts.rs +++ b/vortex-cuda/src/kernel/encodings/date_time_parts.rs @@ -13,7 +13,7 @@ use vortex::array::IntoArray; use vortex::array::arrays::ConstantArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::TemporalArray; -use vortex::array::arrays::primitive::PrimitiveArrayParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::array::buffer::BufferHandle; use vortex::array::match_each_signed_integer_ptype; use vortex::array::validity::Validity; @@ -147,15 +147,15 @@ where { let output_len = days.len(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer: days_buffer, .. } = days.into_data().into_parts(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer: seconds_buffer, .. } = seconds.into_data().into_parts(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer: subseconds_buffer, .. } = subseconds.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs b/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs index 134e5c99374..88f0ad65f96 100644 --- a/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs +++ b/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs @@ -8,11 +8,12 @@ use tracing::instrument; use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::arrays::DecimalArray; -use vortex::array::arrays::primitive::PrimitiveArrayParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::encodings::decimal_byte_parts::DecimalByteParts; -use vortex::encodings::decimal_byte_parts::DecimalBytePartsArrayParts; +use vortex::encodings::decimal_byte_parts::DecimalBytePartsDataParts; use vortex::error::VortexResult; use vortex::error::vortex_bail; +use vortex::error::VortexExpect; use crate::CudaExecutionCtx; use crate::executor::CudaArrayExt; @@ -34,9 +35,12 @@ impl CudaExecute for DecimalBytePartsExecutor { vortex_bail!("cannot downcast to DecimalBytePartsArray") }; - let decimal_dtype = *array.decimal_dtype(); - let DecimalBytePartsArrayParts { msp, .. } = array.into_data().into_parts(); - let PrimitiveArrayParts { + let decimal_dtype = *array + .dtype() + .as_decimal_opt() + .vortex_expect("DecimalBytePartsArray dtype must be decimal"); + let DecimalBytePartsDataParts { msp, .. } = array.into_data().into_parts(); + let PrimitiveDataParts { buffer, ptype, validity, diff --git a/vortex-cuda/src/kernel/encodings/for_.rs b/vortex-cuda/src/kernel/encodings/for_.rs index 57361905c90..7fe9d89d64c 100644 --- a/vortex-cuda/src/kernel/encodings/for_.rs +++ b/vortex-cuda/src/kernel/encodings/for_.rs @@ -12,7 +12,7 @@ use vortex::array::Canonical; use vortex::array::IntoArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::Slice; -use vortex::array::arrays::primitive::PrimitiveArrayParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::array::match_each_integer_ptype; use vortex::array::match_each_native_simd_ptype; use vortex::dtype::NativePType; @@ -52,7 +52,7 @@ impl CudaExecute for FoRExecutor { // Fuse FOR + BP => FFOR if let Some(bitpacked) = array.encoded().as_opt::() { - match_each_integer_ptype!(bitpacked.ptype(), |P| { + match_each_integer_ptype!(bitpacked.ptype(bitpacked.dtype()), |P| { let reference: P = array.reference_scalar().try_into()?; return decode_bitpacked(bitpacked.into_owned(), reference, ctx).await; }) @@ -63,7 +63,7 @@ impl CudaExecute for FoRExecutor { && let Some(bitpacked) = slice_array.child().as_opt::() { let slice_range = slice_array.slice_range().clone(); - let unpacked = match_each_integer_ptype!(bitpacked.ptype(), |P| { + let unpacked = match_each_integer_ptype!(bitpacked.ptype(bitpacked.dtype()), |P| { let reference: P = array.reference_scalar().try_into()?; decode_bitpacked(bitpacked.into_owned(), reference, ctx).await? }); @@ -96,7 +96,7 @@ where // Execute child and copy to device let canonical = array.encoded().clone().execute_cuda(ctx).await?; let primitive = canonical.into_primitive(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer, validity, .. } = primitive.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/encodings/runend.rs b/vortex-cuda/src/kernel/encodings/runend.rs index 6f14ec20c9c..88b64393a5e 100644 --- a/vortex-cuda/src/kernel/encodings/runend.rs +++ b/vortex-cuda/src/kernel/encodings/runend.rs @@ -12,7 +12,7 @@ use vortex::array::Canonical; use vortex::array::IntoArray; use vortex::array::arrays::ConstantArray; use vortex::array::arrays::PrimitiveArray; -use vortex::array::arrays::primitive::PrimitiveArrayParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::array::buffer::BufferHandle; use vortex::array::match_each_native_ptype; use vortex::array::match_each_unsigned_integer_ptype; @@ -21,7 +21,7 @@ use vortex::dtype::NativePType; use vortex::dtype::PType; use vortex::encodings::runend::RunEnd; use vortex::encodings::runend::RunEndArray; -use vortex::encodings::runend::RunEndArrayParts; +use vortex::encodings::runend::RunEndDataParts; use vortex::error::VortexResult; use vortex::error::vortex_bail; use vortex::error::vortex_ensure; @@ -61,7 +61,7 @@ impl CudaExecute for RunEndExecutor { let offset = array.offset(); let output_len = array.len(); - let RunEndArrayParts { ends, values } = array.into_data().into_parts(); + let RunEndDataParts { ends, values, .. } = array.into_data().into_parts(); let values_ptype = PType::try_from(values.dtype())?; let ends_ptype = PType::try_from(ends.dtype())?; @@ -105,14 +105,14 @@ async fn decode_runend_typed() .map_err(|_| vortex_err!("SequenceExecutor can only accept SequenceArray"))?; - let SequenceArrayParts { + let len = array.len(); + let nullability = array.dtype().nullability(); + + let SequenceDataParts { base, multiplier, - len, ptype, - nullability, } = array.into_data().into_parts(); match_each_native_ptype!(ptype, |P| { diff --git a/vortex-cuda/src/kernel/encodings/zigzag.rs b/vortex-cuda/src/kernel/encodings/zigzag.rs index f1feac1bbf3..e7cf89b2181 100644 --- a/vortex-cuda/src/kernel/encodings/zigzag.rs +++ b/vortex-cuda/src/kernel/encodings/zigzag.rs @@ -10,7 +10,7 @@ use tracing::instrument; use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::arrays::PrimitiveArray; -use vortex::array::arrays::primitive::PrimitiveArrayParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::array::match_each_unsigned_integer_ptype; use vortex::dtype::NativePType; use vortex::dtype::PType; @@ -70,7 +70,7 @@ where // Execute child and copy to device let canonical = array.encoded().clone().execute_cuda(ctx).await?; let primitive = canonical.into_primitive(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer, validity, .. } = primitive.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/encodings/zstd.rs b/vortex-cuda/src/kernel/encodings/zstd.rs index a521fe7a27f..f7044697dde 100644 --- a/vortex-cuda/src/kernel/encodings/zstd.rs +++ b/vortex-cuda/src/kernel/encodings/zstd.rs @@ -26,7 +26,7 @@ use vortex::buffer::ByteBuffer; use vortex::dtype::DType; use vortex::encodings::zstd::Zstd; use vortex::encodings::zstd::ZstdArray; -use vortex::encodings::zstd::ZstdArrayParts; +use vortex::encodings::zstd::ZstdDataParts; use vortex::encodings::zstd::ZstdMetadata; use vortex::error::VortexExpect; use vortex::error::VortexResult; @@ -206,17 +206,17 @@ impl CudaExecute for ZstdExecutor { dtype = %_other, "Only Binary/Utf8 ZSTD arrays supported on GPU, falling back to CPU" ); - zstd.decompress(ctx.execution_ctx())?.to_canonical() + Zstd::decompress(&zstd, ctx.execution_ctx())?.to_canonical() } } } } async fn decode_zstd(array: ZstdArray, ctx: &mut CudaExecutionCtx) -> VortexResult { - let ZstdArrayParts { + let dtype = array.dtype().clone(); + let ZstdDataParts { frames, metadata, - dtype, validity, n_rows, dictionary, diff --git a/vortex-cuda/src/kernel/filter/decimal.rs b/vortex-cuda/src/kernel/filter/decimal.rs index 88e07465968..5e468dadee3 100644 --- a/vortex-cuda/src/kernel/filter/decimal.rs +++ b/vortex-cuda/src/kernel/filter/decimal.rs @@ -4,7 +4,7 @@ use cudarc::driver::DeviceRepr; use vortex::array::Canonical; use vortex::array::arrays::DecimalArray; -use vortex::array::arrays::decimal::DecimalArrayParts; +use vortex::array::arrays::decimal::DecimalDataParts; use vortex::dtype::NativeDecimalType; use vortex::error::VortexResult; use vortex::mask::Mask; @@ -18,7 +18,7 @@ pub(super) async fn filter_decimal VortexResult { - let DecimalArrayParts { + let DecimalDataParts { values, validity, decimal_dtype, diff --git a/vortex-cuda/src/kernel/filter/mod.rs b/vortex-cuda/src/kernel/filter/mod.rs index 588c3e07925..4cf8efbc203 100644 --- a/vortex-cuda/src/kernel/filter/mod.rs +++ b/vortex-cuda/src/kernel/filter/mod.rs @@ -19,7 +19,7 @@ use tracing::instrument; use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::arrays::Filter; -use vortex::array::arrays::filter::FilterArrayParts; +use vortex::array::arrays::filter::FilterDataParts; use vortex::array::buffer::BufferHandle; use vortex::array::match_each_decimal_value_type; use vortex::array::match_each_native_simd_ptype; @@ -53,7 +53,7 @@ impl CudaExecute for FilterExecutor { .try_into::() .map_err(|_| vortex_err!("Expected FilterArray"))?; - let FilterArrayParts { child, mask } = filter_array.into_data().into_parts(); + let FilterDataParts { child, mask } = filter_array.into_data().into_parts(); // Early return for trivial cases. match mask { diff --git a/vortex-cuda/src/kernel/filter/primitive.rs b/vortex-cuda/src/kernel/filter/primitive.rs index 68e39edd60f..f45fe3f2994 100644 --- a/vortex-cuda/src/kernel/filter/primitive.rs +++ b/vortex-cuda/src/kernel/filter/primitive.rs @@ -4,7 +4,7 @@ use cudarc::driver::DeviceRepr; use vortex::array::Canonical; use vortex::array::arrays::PrimitiveArray; -use vortex::array::arrays::primitive::PrimitiveArrayParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::dtype::NativePType; use vortex::error::VortexResult; use vortex::mask::Mask; @@ -22,7 +22,7 @@ pub(super) async fn filter_primitive( where T: NativePType + DeviceRepr + CubFilterable + Send + Sync + 'static, { - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer, validity, .. } = array.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/filter/varbinview.rs b/vortex-cuda/src/kernel/filter/varbinview.rs index 143e3f26ce7..8d673180d58 100644 --- a/vortex-cuda/src/kernel/filter/varbinview.rs +++ b/vortex-cuda/src/kernel/filter/varbinview.rs @@ -3,7 +3,7 @@ use vortex::array::Canonical; use vortex::array::arrays::VarBinViewArray; -use vortex::array::arrays::varbinview::VarBinViewArrayParts; +use vortex::array::arrays::varbinview::VarBinViewDataParts; use vortex::error::VortexResult; use vortex::mask::Mask; @@ -15,7 +15,7 @@ pub(super) async fn filter_varbinview( mask: Mask, ctx: &mut CudaExecutionCtx, ) -> VortexResult { - let VarBinViewArrayParts { + let VarBinViewDataParts { views, buffers, validity, diff --git a/vortex-cuda/src/kernel/patches/mod.rs b/vortex-cuda/src/kernel/patches/mod.rs index 9da4ddf8f51..5d833c1c417 100644 --- a/vortex-cuda/src/kernel/patches/mod.rs +++ b/vortex-cuda/src/kernel/patches/mod.rs @@ -12,7 +12,7 @@ pub mod gpu { use cudarc::driver::DeviceRepr; use cudarc::driver::PushKernelArg; use tracing::instrument; -use vortex::array::arrays::primitive::PrimitiveArrayParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::array::patches::Patches; use vortex::array::validity::Validity; use vortex::dtype::NativePType; @@ -67,12 +67,12 @@ pub(crate) async fn execute_patches< let patches_len = indices.len(); let patches_len_u64 = patches_len as u64; - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer: indices_buffer, .. } = indices.into_data().into_parts(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer: values_buffer, .. } = values.into_data().into_parts(); @@ -105,7 +105,7 @@ mod tests { use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; - use vortex::array::arrays::primitive::PrimitiveArrayParts; + use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::array::assert_arrays_eq; use vortex::array::buffer::BufferHandle; use vortex::array::builtins::ArrayBuiltins; @@ -167,7 +167,7 @@ mod tests { ) .unwrap(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer: cuda_buffer, .. } = values.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/slice/mod.rs b/vortex-cuda/src/kernel/slice/mod.rs index b418e0675f7..5326780b5f9 100644 --- a/vortex-cuda/src/kernel/slice/mod.rs +++ b/vortex-cuda/src/kernel/slice/mod.rs @@ -7,7 +7,7 @@ use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::IntoArray; use vortex::array::arrays::Slice; -use vortex::array::arrays::slice::SliceArrayParts; +use vortex::array::arrays::slice::SliceDataParts; use vortex::error::VortexResult; use vortex::error::vortex_err; @@ -33,7 +33,7 @@ impl CudaExecute for SliceExecutor { ) })?; - let SliceArrayParts { child, range } = slice_array.into_data().into_parts(); + let SliceDataParts { child, range } = slice_array.into_data().into_parts(); let child = child.execute_cuda(ctx).await?; match child { diff --git a/vortex-duckdb/src/exporter/decimal.rs b/vortex-duckdb/src/exporter/decimal.rs index 00026eb9f90..765e76272ca 100644 --- a/vortex-duckdb/src/exporter/decimal.rs +++ b/vortex-duckdb/src/exporter/decimal.rs @@ -6,7 +6,7 @@ use std::marker::PhantomData; use num_traits::ToPrimitive; use vortex::array::ExecutionCtx; use vortex::array::arrays::DecimalArray; -use vortex::array::arrays::decimal::DecimalArrayParts; +use vortex::array::arrays::decimal::DecimalDataParts; use vortex::array::match_each_decimal_value_type; use vortex::buffer::Buffer; use vortex::dtype::BigCast; @@ -42,7 +42,7 @@ pub(crate) fn new_exporter( ctx: &mut ExecutionCtx, ) -> VortexResult> { let len = array.len(); - let DecimalArrayParts { + let DecimalDataParts { validity, decimal_dtype, values_type, diff --git a/vortex-duckdb/src/exporter/list.rs b/vortex-duckdb/src/exporter/list.rs index fab62dcc8cb..f2945123a13 100644 --- a/vortex-duckdb/src/exporter/list.rs +++ b/vortex-duckdb/src/exporter/list.rs @@ -8,7 +8,7 @@ use parking_lot::Mutex; use vortex::array::ExecutionCtx; use vortex::array::arrays::ListArray; use vortex::array::arrays::PrimitiveArray; -use vortex::array::arrays::list::ListArrayParts; +use vortex::array::arrays::list::ListDataParts; use vortex::array::match_each_integer_ptype; use vortex::dtype::IntegerPType; use vortex::error::VortexResult; @@ -45,7 +45,7 @@ pub(crate) fn new_exporter( ) -> VortexResult> { let array_len = array.len(); // Cache an `elements` vector up front so that future exports can reference it. - let ListArrayParts { + let ListDataParts { elements, offsets, validity, diff --git a/vortex-duckdb/src/exporter/list_view.rs b/vortex-duckdb/src/exporter/list_view.rs index 4b76f534ff2..226f399c92f 100644 --- a/vortex-duckdb/src/exporter/list_view.rs +++ b/vortex-duckdb/src/exporter/list_view.rs @@ -8,7 +8,7 @@ use parking_lot::Mutex; use vortex::array::ExecutionCtx; use vortex::array::arrays::ListViewArray; use vortex::array::arrays::PrimitiveArray; -use vortex::array::arrays::listview::ListViewArrayParts; +use vortex::array::arrays::listview::ListViewDataParts; use vortex::array::match_each_integer_ptype; use vortex::dtype::DType; use vortex::dtype::IntegerPType; @@ -47,7 +47,7 @@ pub(crate) fn new_exporter( ctx: &mut ExecutionCtx, ) -> VortexResult> { let len = array.len(); - let ListViewArrayParts { + let ListViewDataParts { elements_dtype, elements, offsets, diff --git a/vortex-duckdb/src/exporter/run_end.rs b/vortex-duckdb/src/exporter/run_end.rs index 0d05cf67a30..af21438130d 100644 --- a/vortex-duckdb/src/exporter/run_end.rs +++ b/vortex-duckdb/src/exporter/run_end.rs @@ -11,7 +11,7 @@ use vortex::array::search_sorted::SearchSorted; use vortex::array::search_sorted::SearchSortedSide; use vortex::dtype::IntegerPType; use vortex::encodings::runend::RunEndArray; -use vortex::encodings::runend::RunEndArrayParts; +use vortex::encodings::runend::RunEndDataParts; use vortex::error::VortexExpect; use vortex::error::VortexResult; @@ -38,7 +38,7 @@ pub(crate) fn new_exporter( ctx: &mut ExecutionCtx, ) -> VortexResult> { let offset = array.offset(); - let RunEndArrayParts { ends, values } = array.into_data().into_parts(); + let RunEndDataParts { ends, values, .. } = array.into_data().into_parts(); let ends = ends.execute::(ctx)?; let values_exporter = new_array_exporter(values.clone(), cache, ctx)?; diff --git a/vortex-duckdb/src/exporter/struct_.rs b/vortex-duckdb/src/exporter/struct_.rs index 8c07ada0533..19ac59c69d9 100644 --- a/vortex-duckdb/src/exporter/struct_.rs +++ b/vortex-duckdb/src/exporter/struct_.rs @@ -5,7 +5,7 @@ use vortex::array::ExecutionCtx; use vortex::array::IntoArray; use vortex::array::arrays::BoolArray; use vortex::array::arrays::StructArray; -use vortex::array::arrays::struct_::StructArrayParts; +use vortex::array::arrays::struct_::StructDataParts; use vortex::array::builtins::ArrayBuiltins; use vortex::error::VortexResult; @@ -27,12 +27,12 @@ pub(crate) fn new_exporter( ctx: &mut ExecutionCtx, ) -> VortexResult> { let len = array.len(); - let StructArrayParts { + let StructDataParts { validity, struct_fields, fields, .. - } = array.into_parts(); + } = array.into_data().into_parts(); let validity = validity.to_array(len).execute::(ctx)?; if validity.to_bit_buffer().true_count() == 0 { diff --git a/vortex-duckdb/src/exporter/varbinview.rs b/vortex-duckdb/src/exporter/varbinview.rs index f0667ee6144..23142b5c4cb 100644 --- a/vortex-duckdb/src/exporter/varbinview.rs +++ b/vortex-duckdb/src/exporter/varbinview.rs @@ -8,7 +8,7 @@ use vortex::array::ExecutionCtx; use vortex::array::arrays::VarBinViewArray; use vortex::array::arrays::varbinview::BinaryView; use vortex::array::arrays::varbinview::Inlined; -use vortex::array::arrays::varbinview::VarBinViewArrayParts; +use vortex::array::arrays::varbinview::VarBinViewDataParts; use vortex::buffer::Buffer; use vortex::buffer::ByteBuffer; use vortex::error::VortexResult; @@ -32,7 +32,7 @@ pub(crate) fn new_exporter( ctx: &mut ExecutionCtx, ) -> VortexResult> { let len = array.len(); - let VarBinViewArrayParts { + let VarBinViewDataParts { validity, dtype, views, diff --git a/vortex-python/src/arrays/py/array.rs b/vortex-python/src/arrays/py/array.rs index d60c9469e59..858f9280305 100644 --- a/vortex-python/src/arrays/py/array.rs +++ b/vortex-python/src/arrays/py/array.rs @@ -13,6 +13,7 @@ use vortex::array::ArrayParts; use vortex::array::ArrayRef; use vortex::array::IntoArray; use vortex::array::stats::ArrayStats; +use vortex::array::stats::StatsSet; use vortex::dtype::DType; use crate::arrays::py::PyPythonArray; @@ -65,8 +66,9 @@ impl IntoArray for PythonArray { let vtable = self.vtable.clone(); let dtype = self.dtype.clone(); let len = self.len; - let stats = self.stats.clone(); - Array::try_from_parts(ArrayParts::new(vtable, dtype, len, self).with_stats(stats)) + let stats = StatsSet::from(self.stats.clone()); + Array::try_from_parts(ArrayParts::new(vtable, dtype, len, self)) + .map(|array| array.with_stats_set(stats)) .expect("PythonArray metadata extracted from PyPythonArray must be valid") .into_array() } From c3cd511410bbbc6cb8779cc8a888d05dd5f89eb0 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Fri, 3 Apr 2026 09:53:44 -0400 Subject: [PATCH 06/15] Advanced frame parsing Signed-off-by: Nicholas Gates --- encodings/alp/src/alp/mod.rs | 7 ++-- encodings/alp/src/alp_rd/array.rs | 7 ++-- encodings/bytebool/src/array.rs | 5 ++- encodings/datetime-parts/src/array.rs | 12 +++--- encodings/datetime-parts/src/lib.rs | 7 ++-- .../src/decimal_byte_parts/mod.rs | 8 ++-- encodings/fastlanes/src/delta/vtable/mod.rs | 7 ++-- encodings/fastlanes/src/rle/vtable/mod.rs | 7 ++-- encodings/fsst/src/array.rs | 19 +++++---- encodings/runend/src/array.rs | 12 +++--- encodings/runend/src/lib.rs | 7 ++-- encodings/sequence/src/array.rs | 19 ++++----- encodings/sparse/public-api.lock | 40 +++++++++---------- encodings/sparse/src/lib.rs | 16 ++++---- encodings/zigzag/src/array.rs | 5 ++- vortex-array/public-api.lock | 10 ----- vortex-array/src/arrays/bool/vtable/mod.rs | 12 +++--- vortex-array/src/arrays/chunked/vtable/mod.rs | 5 ++- vortex-array/src/arrays/decimal/vtable/mod.rs | 12 +++--- vortex-array/src/arrays/dict/array.rs | 7 ++-- vortex-array/src/arrays/dict/vtable/mod.rs | 12 +++--- .../src/arrays/extension/vtable/mod.rs | 8 +++- .../src/arrays/fixed_size_list/vtable/mod.rs | 8 +++- vortex-array/src/arrays/list/vtable/mod.rs | 12 +++--- .../src/arrays/listview/vtable/mod.rs | 12 +++--- vortex-array/src/arrays/masked/vtable/mod.rs | 5 ++- vortex-array/src/arrays/null/mod.rs | 7 +++- vortex-array/src/arrays/patched/array.rs | 25 ------------ vortex-array/src/arrays/patched/vtable/mod.rs | 32 +++++++++++---- .../src/arrays/primitive/vtable/mod.rs | 8 +++- vortex-array/src/arrays/struct_/array.rs | 22 ---------- vortex-array/src/arrays/struct_/vtable/mod.rs | 26 ++++++++++-- vortex-array/src/arrays/varbin/array.rs | 15 ------- vortex-array/src/arrays/varbin/vtable/mod.rs | 26 ++++++++---- vortex-array/src/arrays/varbinview/array.rs | 16 -------- .../src/arrays/varbinview/vtable/mod.rs | 22 ++++++++-- vortex-array/src/arrays/variant/vtable/mod.rs | 7 +++- vortex-array/src/test_harness.rs | 11 +---- 38 files changed, 240 insertions(+), 258 deletions(-) diff --git a/encodings/alp/src/alp/mod.rs b/encodings/alp/src/alp/mod.rs index b68d58b2ad3..7d5049b59e3 100644 --- a/encodings/alp/src/alp/mod.rs +++ b/encodings/alp/src/alp/mod.rs @@ -22,7 +22,7 @@ mod rules; #[cfg(test)] mod tests { - use vortex_array::ProstMetadata; + use prost::Message; use vortex_array::dtype::PType; use vortex_array::patches::PatchesMetadata; use vortex_array::test_harness::check_metadata; @@ -34,7 +34,7 @@ mod tests { fn test_alp_metadata() { check_metadata( "alp.metadata", - ProstMetadata(ALPMetadata { + &ALPMetadata { patches: Some(PatchesMetadata::new( usize::MAX, usize::MAX, @@ -45,7 +45,8 @@ mod tests { )), exp_e: u32::MAX, exp_f: u32::MAX, - }), + } + .encode_to_vec(), ); } } diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index 8c1cd630ce6..6922e40733e 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -651,8 +651,8 @@ impl ValidityChild for ALPRD { #[cfg(test)] mod test { + use prost::Message; use rstest::rstest; - use vortex_array::ProstMetadata; use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -696,7 +696,7 @@ mod test { fn test_alprd_metadata() { check_metadata( "alprd.metadata", - ProstMetadata(ALPRDMetadata { + &ALPRDMetadata { right_bit_width: u32::MAX, patches: Some(PatchesMetadata::new( usize::MAX, @@ -709,7 +709,8 @@ mod test { dict: Vec::new(), left_parts_ptype: PType::U64 as i32, dict_len: 8, - }), + } + .encode_to_vec(), ); } } diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index ebde50f0a90..b9bf9aa3a5f 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -10,7 +10,6 @@ use vortex_array::ArrayId; use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; -use vortex_array::EmptyMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; @@ -96,7 +95,9 @@ impl VTable for ByteBool { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - ::deserialize(metadata)?; + if !metadata.is_empty() { + vortex_bail!("ByteBoolArray expects empty metadata, got {} bytes", metadata.len()); + } let validity = if children.is_empty() { Validity::from(dtype.nullability()) } else if children.len() == 1 { diff --git a/encodings/datetime-parts/src/array.rs b/encodings/datetime-parts/src/array.rs index e4163ab85a8..9783b665953 100644 --- a/encodings/datetime-parts/src/array.rs +++ b/encodings/datetime-parts/src/array.rs @@ -3,6 +3,7 @@ use std::fmt::Debug; +use prost::Message; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; @@ -10,13 +11,10 @@ use vortex_array::ArrayId; use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; -use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; -use vortex_array::SerializeMetadata; use vortex_array::arrays::TemporalArray; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; @@ -119,12 +117,12 @@ impl VTable for DateTimeParts { fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some( - ProstMetadata(DateTimePartsMetadata { + DateTimePartsMetadata { days_ptype: PType::try_from(array.days().dtype())? as i32, seconds_ptype: PType::try_from(array.seconds().dtype())? as i32, subseconds_ptype: PType::try_from(array.subseconds().dtype())? as i32, - }) - .serialize(), + } + .encode_to_vec(), )) } @@ -137,7 +135,7 @@ impl VTable for DateTimeParts { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - let metadata = ProstMetadata::::deserialize(metadata)?; + let metadata = DateTimePartsMetadata::decode(metadata)?; if children.len() != 3 { vortex_bail!( "Expected 3 children for datetime-parts encoding, found {}", diff --git a/encodings/datetime-parts/src/lib.rs b/encodings/datetime-parts/src/lib.rs index 93032b0b97d..091f4058c14 100644 --- a/encodings/datetime-parts/src/lib.rs +++ b/encodings/datetime-parts/src/lib.rs @@ -30,7 +30,7 @@ pub fn initialize(session: &mut VortexSession) { #[cfg(test)] mod test { - use vortex_array::ProstMetadata; + use prost::Message; use vortex_array::dtype::PType; use vortex_array::test_harness::check_metadata; @@ -41,11 +41,12 @@ mod test { fn test_datetimeparts_metadata() { check_metadata( "datetimeparts.metadata", - ProstMetadata(DateTimePartsMetadata { + &DateTimePartsMetadata { days_ptype: PType::I64 as i32, seconds_ptype: PType::I64 as i32, subseconds_ptype: PType::I64 as i32, - }), + } + .encode_to_vec(), ); } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs index a272f4d7f1b..67d243df1b9 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs @@ -17,8 +17,6 @@ use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; -use vortex_array::SerializeMetadata; use vortex_array::arrays::DecimalArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; @@ -102,11 +100,11 @@ impl VTable for DecimalByteParts { fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some( - ProstMetadata(DecimalBytesPartsMetadata { + DecimalBytesPartsMetadata { zeroth_child_ptype: PType::try_from(array.msp().dtype())? as i32, lower_part_count: 0, - }) - .serialize(), + } + .encode_to_vec(), )) } diff --git a/encodings/fastlanes/src/delta/vtable/mod.rs b/encodings/fastlanes/src/delta/vtable/mod.rs index c4bedc8d9a0..ba31e1a13c8 100644 --- a/encodings/fastlanes/src/delta/vtable/mod.rs +++ b/encodings/fastlanes/src/delta/vtable/mod.rs @@ -199,7 +199,7 @@ impl Delta { #[cfg(test)] mod tests { - use vortex_array::ProstMetadata; + use prost::Message; use vortex_array::test_harness::check_metadata; use super::DeltaMetadata; @@ -209,10 +209,11 @@ mod tests { fn test_delta_metadata() { check_metadata( "delta.metadata", - ProstMetadata(DeltaMetadata { + &DeltaMetadata { offset: u32::MAX, deltas_len: u64::MAX, - }), + } + .encode_to_vec(), ); } } diff --git a/encodings/fastlanes/src/rle/vtable/mod.rs b/encodings/fastlanes/src/rle/vtable/mod.rs index 8dc93872ba2..c47f7adf5df 100644 --- a/encodings/fastlanes/src/rle/vtable/mod.rs +++ b/encodings/fastlanes/src/rle/vtable/mod.rs @@ -244,7 +244,7 @@ impl RLE { #[cfg(test)] mod tests { - use vortex_array::ProstMetadata; + use prost::Message; use vortex_array::test_harness::check_metadata; use super::RLEMetadata; @@ -254,14 +254,15 @@ mod tests { fn test_rle_metadata() { check_metadata( "rle.metadata", - ProstMetadata(RLEMetadata { + &RLEMetadata { values_len: u64::MAX, indices_len: u64::MAX, indices_ptype: i32::MAX, values_idx_offsets_len: u64::MAX, values_idx_offsets_ptype: i32::MAX, offset: u64::MAX, - }), + } + .encode_to_vec(), ); } } diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index 62874cfc3dc..e267283a4cd 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -568,9 +568,10 @@ impl ValidityChild for FSST { mod test { use fsst::Compressor; use fsst::Symbol; + use prost::Message; + use vortex_array::DynVTable; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; - use vortex_array::ProstMetadata; use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::VarBinViewArray; @@ -579,7 +580,6 @@ mod test { use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::test_harness::check_metadata; - use vortex_array::vtable::VTable; use vortex_buffer::Buffer; use vortex_error::VortexError; @@ -592,10 +592,11 @@ mod test { fn test_fsst_metadata() { check_metadata( "fsst.metadata", - ProstMetadata(FSSTMetadata { + &FSSTMetadata { uncompressed_lengths_ptype: PType::U64 as i32, codes_offsets_ptype: PType::I32 as i32, - }), + } + .encode_to_vec(), ); } @@ -640,10 +641,12 @@ mod test { fsst_array.uncompressed_lengths().clone(), ]; - let fsst = FSST::build( + let fsst = DynVTable::build( + &FSST, + FSST::ID, &DType::Utf8(Nullability::NonNullable), 2, - &ProstMetadata(FSSTMetadata { + &FSSTMetadata { uncompressed_lengths_ptype: fsst_array .uncompressed_lengths() .dtype() @@ -651,9 +654,11 @@ mod test { .into(), // Legacy array did not store this field, use Protobuf default of 0. codes_offsets_ptype: 0, - }), + } + .encode_to_vec(), &buffers, &children.as_slice(), + &LEGACY_SESSION, ) .unwrap(); diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index 8271e34d349..35c1b560626 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -4,6 +4,7 @@ use std::fmt::Debug; use std::hash::Hash; +use prost::Message; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; @@ -11,13 +12,10 @@ use vortex_array::ArrayId; use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; -use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; -use vortex_array::SerializeMetadata; use vortex_array::arrays::Primitive; use vortex_array::arrays::VarBinViewArray; use vortex_array::buffer::BufferHandle; @@ -105,13 +103,13 @@ impl VTable for RunEnd { fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some( - ProstMetadata(RunEndMetadata { + RunEndMetadata { ends_ptype: PType::try_from(array.ends().dtype()) .vortex_expect("Must be a valid PType") as i32, num_runs: array.ends().len() as u64, offset: array.offset() as u64, - }) - .serialize(), + } + .encode_to_vec(), )) } @@ -124,7 +122,7 @@ impl VTable for RunEnd { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - let metadata = ProstMetadata::::deserialize(metadata)?; + let metadata = RunEndMetadata::decode(metadata)?; let ends_dtype = DType::Primitive(metadata.ends_ptype(), Nullability::NonNullable); let runs = usize::try_from(metadata.num_runs).vortex_expect("Must be a valid usize"); let ends = children.get(0, &ends_dtype, runs)?; diff --git a/encodings/runend/src/lib.rs b/encodings/runend/src/lib.rs index 13ca4f2e596..71c706049b3 100644 --- a/encodings/runend/src/lib.rs +++ b/encodings/runend/src/lib.rs @@ -58,7 +58,7 @@ pub fn initialize(session: &mut VortexSession) { #[cfg(test)] mod tests { - use vortex_array::ProstMetadata; + use prost::Message; use vortex_array::dtype::PType; use vortex_array::test_harness::check_metadata; @@ -69,11 +69,12 @@ mod tests { fn test_runend_metadata() { check_metadata( "runend.metadata", - ProstMetadata(RunEndMetadata { + &RunEndMetadata { ends_ptype: PType::U64 as i32, num_runs: u64::MAX, offset: u64::MAX, - }), + } + .encode_to_vec(), ); } } diff --git a/encodings/sequence/src/array.rs b/encodings/sequence/src/array.rs index d8458499551..42d69821700 100644 --- a/encodings/sequence/src/array.rs +++ b/encodings/sequence/src/array.rs @@ -4,17 +4,15 @@ use std::hash::Hash; use num_traits::cast::FromPrimitive; +use prost::Message; use vortex_array::Array; use vortex_array::ArrayId; use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; -use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::Precision; -use vortex_array::ProstMetadata; -use vortex_array::SerializeMetadata; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; @@ -51,7 +49,7 @@ use crate::rules::RULES; vtable!(Sequence, Sequence, SequenceData); #[derive(Clone, prost::Message)] -pub struct ProstSequenceMetadata { +pub struct SequenceMetadata { #[prost(message, tag = "1")] base: Option, #[prost(message, tag = "2")] @@ -248,12 +246,12 @@ impl VTable for Sequence { } fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - let prost = ProstMetadata(ProstSequenceMetadata { + let metadata = SequenceMetadata { base: Some((&array.base()).into()), multiplier: Some((&array.multiplier()).into()), - }); + }; - Ok(Some(prost.serialize())) + Ok(Some(metadata.encode_to_vec())) } fn deserialize( @@ -275,14 +273,13 @@ impl VTable for Sequence { "SequenceArray expects 0 children, got {}", children.len() ); - let prost = - as DeserializeMetadata>::deserialize(metadata)?; + let metadata = SequenceMetadata::decode(metadata)?; let ptype = dtype.as_ptype(); // We go via Scalar to validate that the value is valid for the ptype. let base = Scalar::from_proto_value( - prost + metadata .base .as_ref() .ok_or_else(|| vortex_err!("base required"))?, @@ -294,7 +291,7 @@ impl VTable for Sequence { .vortex_expect("sequence array base should be a non-nullable primitive"); let multiplier = Scalar::from_proto_value( - prost + metadata .multiplier .as_ref() .ok_or_else(|| vortex_err!("multiplier required"))?, diff --git a/encodings/sparse/public-api.lock b/encodings/sparse/public-api.lock index 74d820647b4..b937d3df263 100644 --- a/encodings/sparse/public-api.lock +++ b/encodings/sparse/public-api.lock @@ -1,25 +1,5 @@ pub mod vortex_sparse -#[repr(C)] pub struct vortex_sparse::ProstPatchesMetadata - -impl core::clone::Clone for vortex_sparse::ProstPatchesMetadata - -pub fn vortex_sparse::ProstPatchesMetadata::clone(&self) -> vortex_sparse::ProstPatchesMetadata - -impl core::default::Default for vortex_sparse::ProstPatchesMetadata - -pub fn vortex_sparse::ProstPatchesMetadata::default() -> Self - -impl core::fmt::Debug for vortex_sparse::ProstPatchesMetadata - -pub fn vortex_sparse::ProstPatchesMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -impl prost::message::Message for vortex_sparse::ProstPatchesMetadata - -pub fn vortex_sparse::ProstPatchesMetadata::clear(&mut self) - -pub fn vortex_sparse::ProstPatchesMetadata::encoded_len(&self) -> usize - pub struct vortex_sparse::Sparse impl vortex_sparse::Sparse @@ -138,4 +118,24 @@ impl core::fmt::Debug for vortex_sparse::SparseData pub fn vortex_sparse::SparseData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +#[repr(C)] pub struct vortex_sparse::SparseMetadata + +impl core::clone::Clone for vortex_sparse::SparseMetadata + +pub fn vortex_sparse::SparseMetadata::clone(&self) -> vortex_sparse::SparseMetadata + +impl core::default::Default for vortex_sparse::SparseMetadata + +pub fn vortex_sparse::SparseMetadata::default() -> Self + +impl core::fmt::Debug for vortex_sparse::SparseMetadata + +pub fn vortex_sparse::SparseMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + +impl prost::message::Message for vortex_sparse::SparseMetadata + +pub fn vortex_sparse::SparseMetadata::clear(&mut self) + +pub fn vortex_sparse::SparseMetadata::encoded_len(&self) -> usize + pub type vortex_sparse::SparseArray = vortex_array::array::typed::Array diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index a25dc92e437..c192a6a0667 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -59,7 +59,7 @@ vtable!(Sparse, Sparse, SparseData); #[derive(Clone, prost::Message)] #[repr(C)] -pub struct ProstPatchesMetadata { +pub struct SparseMetadata { #[prost(message, required, tag = "1")] patches: PatchesMetadata, } @@ -111,10 +111,10 @@ impl VTable for Sparse { fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { let patches = array.patches().to_metadata(array.len(), array.dtype())?; - let prost_patches = ProstPatchesMetadata { patches }; + let metadata = SparseMetadata { patches }; // Note that we DO NOT serialize the fill value since that is stored in the buffers. - Ok(Some(prost_patches.encode_to_vec())) + Ok(Some(metadata.encode_to_vec())) } fn deserialize( @@ -126,7 +126,7 @@ impl VTable for Sparse { children: &dyn ArrayChildren, session: &VortexSession, ) -> VortexResult { - let prost_patches = ProstPatchesMetadata::decode(metadata)?; + let metadata = SparseMetadata::decode(metadata)?; // Once we have the patches metadata, we need to get the fill value from the buffers. @@ -147,15 +147,15 @@ impl VTable for Sparse { let patch_indices = children.get( 0, - &prost_patches.patches.indices_dtype()?, - prost_patches.patches.len()?, + &metadata.patches.indices_dtype()?, + metadata.patches.len()?, )?; - let patch_values = children.get(1, dtype, prost_patches.patches.len()?)?; + let patch_values = children.get(1, dtype, metadata.patches.len()?)?; SparseData::try_new_from_patches( Patches::new( len, - prost_patches.patches.offset()?, + metadata.patches.offset()?, patch_indices, patch_values, None, diff --git a/encodings/zigzag/src/array.rs b/encodings/zigzag/src/array.rs index 6b36d7e6c23..29122577e6b 100644 --- a/encodings/zigzag/src/array.rs +++ b/encodings/zigzag/src/array.rs @@ -8,7 +8,6 @@ use vortex_array::ArrayId; use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; -use vortex_array::EmptyMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; @@ -96,7 +95,9 @@ impl VTable for ZigZag { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - ::deserialize(metadata)?; + if !metadata.is_empty() { + vortex_bail!("ZigZagArray expects empty metadata, got {} bytes", metadata.len()); + } if children.len() != 1 { vortex_bail!("Expected 1 child, got {}", children.len()); } diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index e4ead21dddb..1ef22101ebe 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -3022,8 +3022,6 @@ pub fn vortex_array::arrays::patched::PatchedArray::patch_indices(&self) -> &vor pub fn vortex_array::arrays::patched::PatchedArray::patch_values(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::patched::PatchedArray::validate_against_outer(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - impl vortex_array::arrays::patched::PatchedArray pub fn vortex_array::arrays::patched::PatchedArray::from_array_and_patches(inner: vortex_array::ArrayRef, patches: &vortex_array::patches::Patches, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -3950,8 +3948,6 @@ pub fn vortex_array::arrays::struct_::StructData::unmasked_fields(&self) -> allo pub fn vortex_array::arrays::struct_::StructData::validate(fields: &[vortex_array::ArrayRef], dtype: &vortex_array::dtype::StructFields, length: usize, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::struct_::StructData::validate_against_outer(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - pub fn vortex_array::arrays::struct_::StructData::validity(&self) -> vortex_array::validity::Validity impl vortex_array::arrays::struct_::StructData @@ -4154,8 +4150,6 @@ pub fn vortex_array::arrays::varbin::VarBinData::into_parts(self) -> (vortex_arr pub fn vortex_array::arrays::varbin::VarBinData::offset_at(&self, index: usize) -> usize -pub fn vortex_array::arrays::varbin::VarBinData::validate_against_outer(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - impl core::clone::Clone for vortex_array::arrays::varbin::VarBinData pub fn vortex_array::arrays::varbin::VarBinData::clone(&self) -> vortex_array::arrays::varbin::VarBinData @@ -4534,8 +4528,6 @@ pub fn vortex_array::arrays::varbinview::VarBinViewData::try_new_handle(views: v pub fn vortex_array::arrays::varbinview::VarBinViewData::validate(views: &vortex_buffer::buffer::Buffer, buffers: &alloc::sync::Arc<[vortex_buffer::ByteBuffer]>, dtype: &vortex_array::dtype::DType, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::varbinview::VarBinViewData::validate_against_outer(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - pub fn vortex_array::arrays::varbinview::VarBinViewData::validity(&self) -> vortex_array::validity::Validity pub fn vortex_array::arrays::varbinview::VarBinViewData::validity_mask(&self) -> vortex_mask::Mask @@ -5908,8 +5900,6 @@ pub fn vortex_array::arrays::patched::PatchedArray::patch_indices(&self) -> &vor pub fn vortex_array::arrays::patched::PatchedArray::patch_values(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::patched::PatchedArray::validate_against_outer(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - impl vortex_array::arrays::patched::PatchedArray pub fn vortex_array::arrays::patched::PatchedArray::from_array_and_patches(inner: vortex_array::ArrayRef, patches: &vortex_array::patches::Patches, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult diff --git a/vortex-array/src/arrays/bool/vtable/mod.rs b/vortex-array/src/arrays/bool/vtable/mod.rs index 5c192cdec50..0842a90a398 100644 --- a/vortex-array/src/arrays/bool/vtable/mod.rs +++ b/vortex-array/src/arrays/bool/vtable/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use prost::Message; use kernel::PARENT_KERNELS; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -10,11 +11,8 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; -use crate::DeserializeMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; -use crate::ProstMetadata; -use crate::SerializeMetadata; use crate::array::Array; use crate::array::ArrayView; use crate::array::VTable; @@ -89,10 +87,10 @@ impl VTable for Bool { fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { assert!(array.offset < 8, "Offset must be <8, got {}", array.offset); Ok(Some( - ProstMetadata(BoolMetadata { + BoolMetadata { offset: u32::try_from(array.offset).vortex_expect("checked"), - }) - .serialize(), + } + .encode_to_vec(), )) } @@ -125,7 +123,7 @@ impl VTable for Bool { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - let metadata = ProstMetadata::::deserialize(metadata)?; + let metadata = BoolMetadata::decode(metadata)?; if buffers.len() != 1 { vortex_bail!("Expected 1 buffer, got {}", buffers.len()); } diff --git a/vortex-array/src/arrays/chunked/vtable/mod.rs b/vortex-array/src/arrays/chunked/vtable/mod.rs index 0c9aaca6ef2..93b5c96a600 100644 --- a/vortex-array/src/arrays/chunked/vtable/mod.rs +++ b/vortex-array/src/arrays/chunked/vtable/mod.rs @@ -11,7 +11,6 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::Canonical; -use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::IntoArray; @@ -135,7 +134,9 @@ impl VTable for Chunked { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - ::deserialize(metadata)?; + if !metadata.is_empty() { + vortex_bail!("ChunkedArray expects empty metadata, got {} bytes", metadata.len()); + } if children.is_empty() { vortex_bail!("Chunked array needs at least one child"); } diff --git a/vortex-array/src/arrays/decimal/vtable/mod.rs b/vortex-array/src/arrays/decimal/vtable/mod.rs index bd24cd313ba..fba5962b82c 100644 --- a/vortex-array/src/arrays/decimal/vtable/mod.rs +++ b/vortex-array/src/arrays/decimal/vtable/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use prost::Message; use kernel::PARENT_KERNELS; use vortex_buffer::Alignment; use vortex_error::VortexResult; @@ -10,11 +11,8 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; -use crate::DeserializeMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; -use crate::ProstMetadata; -use crate::SerializeMetadata; use crate::array::Array; use crate::array::ArrayView; use crate::array::VTable; @@ -91,10 +89,10 @@ impl VTable for Decimal { fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some( - ProstMetadata(DecimalMetadata { + DecimalMetadata { values_type: array.values_type() as i32, - }) - .serialize(), + } + .encode_to_vec(), )) } @@ -127,7 +125,7 @@ impl VTable for Decimal { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - let metadata = ProstMetadata::::deserialize(metadata)?; + let metadata = DecimalMetadata::decode(metadata)?; if buffers.len() != 1 { vortex_bail!("Expected 1 buffer, got {}", buffers.len()); } diff --git a/vortex-array/src/arrays/dict/array.rs b/vortex-array/src/arrays/dict/array.rs index ec0ce9e26a5..a347862c601 100644 --- a/vortex-array/src/arrays/dict/array.rs +++ b/vortex-array/src/arrays/dict/array.rs @@ -448,18 +448,19 @@ mod test { #[cfg_attr(miri, ignore)] #[test] fn test_dict_metadata() { + use prost::Message; use super::DictMetadata; - use crate::ProstMetadata; use crate::test_harness::check_metadata; check_metadata( "dict.metadata", - ProstMetadata(DictMetadata { + &DictMetadata { codes_ptype: PType::U64 as i32, values_len: u32::MAX, is_nullable_codes: None, all_values_referenced: None, - }), + } + .encode_to_vec(), ); } } diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index d817a1256bd..fa4f243b091 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use prost::Message; use kernel::PARENT_KERNELS; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -19,10 +20,7 @@ use super::take_canonical; use crate::AnyCanonical; use crate::ArrayRef; use crate::Canonical; -use crate::DeserializeMetadata; use crate::Precision; -use crate::ProstMetadata; -use crate::SerializeMetadata; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; @@ -98,7 +96,7 @@ impl VTable for Dict { fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some( - ProstMetadata(DictMetadata { + DictMetadata { codes_ptype: PType::try_from(array.codes().dtype())? as i32, values_len: u32::try_from(array.values().len()).map_err(|_| { vortex_err!( @@ -108,8 +106,8 @@ impl VTable for Dict { })?, is_nullable_codes: Some(array.codes().dtype().is_nullable()), all_values_referenced: Some(array.all_values_referenced), - }) - .serialize(), + } + .encode_to_vec(), )) } @@ -123,7 +121,7 @@ impl VTable for Dict { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - let metadata = ProstMetadata::::deserialize(metadata)?; + let metadata = DictMetadata::decode(metadata)?; if children.len() != 2 { vortex_bail!( "Expected 2 children for dict encoding, found {}", diff --git a/vortex-array/src/arrays/extension/vtable/mod.rs b/vortex-array/src/arrays/extension/vtable/mod.rs index c888da35eee..3ef3bc05617 100644 --- a/vortex-array/src/arrays/extension/vtable/mod.rs +++ b/vortex-array/src/arrays/extension/vtable/mod.rs @@ -13,7 +13,6 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; -use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; @@ -112,7 +111,12 @@ impl VTable for Extension { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - ::deserialize(metadata)?; + if !metadata.is_empty() { + vortex_bail!( + "ExtensionArray expects empty metadata, got {} bytes", + metadata.len() + ); + } let DType::Extension(ext_dtype) = dtype else { vortex_bail!("Not an extension DType"); }; diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs index 59bd04e278b..ab3e468c187 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs @@ -10,7 +10,6 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; -use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; @@ -134,7 +133,12 @@ impl VTable for FixedSizeList { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - ::deserialize(metadata)?; + if !metadata.is_empty() { + vortex_bail!( + "FixedSizeListArray expects empty metadata, got {} bytes", + metadata.len() + ); + } vortex_ensure!( buffers.is_empty(), "`FixedSizeList::build` expects no buffers" diff --git a/vortex-array/src/arrays/list/vtable/mod.rs b/vortex-array/src/arrays/list/vtable/mod.rs index fb267eff690..dd999ee3419 100644 --- a/vortex-array/src/arrays/list/vtable/mod.rs +++ b/vortex-array/src/arrays/list/vtable/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use prost::Message; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -12,7 +13,6 @@ use crate::ExecutionCtx; use crate::ExecutionResult; use crate::IntoArray; use crate::Precision; -use crate::ProstMetadata; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; @@ -29,8 +29,6 @@ use crate::dtype::Nullability; use crate::dtype::PType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; -use crate::metadata::DeserializeMetadata; -use crate::metadata::SerializeMetadata; use crate::serde::ArrayChildren; use crate::validity::Validity; use crate::vtable; @@ -90,11 +88,11 @@ impl VTable for List { fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some( - ProstMetadata(ListMetadata { + ListMetadata { elements_len: array.elements().len() as u64, offset_ptype: PType::try_from(array.offsets().dtype())? as i32, - }) - .serialize(), + } + .encode_to_vec(), )) } @@ -127,7 +125,7 @@ impl VTable for List { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - let metadata = ProstMetadata::::deserialize(metadata)?; + let metadata = ListMetadata::decode(metadata)?; let validity = if children.len() == 2 { Validity::from(dtype.nullability()) } else if children.len() == 3 { diff --git a/vortex-array/src/arrays/listview/vtable/mod.rs b/vortex-array/src/arrays/listview/vtable/mod.rs index cc87352ec65..9dae3483a16 100644 --- a/vortex-array/src/arrays/listview/vtable/mod.rs +++ b/vortex-array/src/arrays/listview/vtable/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use prost::Message; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -8,12 +9,9 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; -use crate::DeserializeMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; -use crate::ProstMetadata; -use crate::SerializeMetadata; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; @@ -90,12 +88,12 @@ impl VTable for ListView { fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some( - ProstMetadata(ListViewMetadata { + ListViewMetadata { elements_len: array.elements().len() as u64, offset_ptype: PType::try_from(array.offsets().dtype())? as i32, size_ptype: PType::try_from(array.sizes().dtype())? as i32, - }) - .serialize(), + } + .encode_to_vec(), )) } @@ -128,7 +126,7 @@ impl VTable for ListView { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - let metadata = ProstMetadata::::deserialize(metadata)?; + let metadata = ListViewMetadata::decode(metadata)?; vortex_ensure!( buffers.is_empty(), "`ListViewArray::build` expects no buffers" diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index dadd023e3d2..3e7b16f48c7 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -12,7 +12,6 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::Canonical; -use crate::EmptyMetadata; use crate::IntoArray; use crate::Precision; use crate::array::Array; @@ -102,7 +101,9 @@ impl VTable for Masked { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - ::deserialize(metadata)?; + if !metadata.is_empty() { + vortex_bail!("MaskedArray expects empty metadata, got {} bytes", metadata.len()); + } if !buffers.is_empty() { vortex_bail!("Expected 0 buffer, got {}", buffers.len()); } diff --git a/vortex-array/src/arrays/null/mod.rs b/vortex-array/src/arrays/null/mod.rs index 2264116d584..3319305a020 100644 --- a/vortex-array/src/arrays/null/mod.rs +++ b/vortex-array/src/arrays/null/mod.rs @@ -8,7 +8,6 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; -use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; @@ -101,7 +100,11 @@ impl VTable for Null { _children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - ::deserialize(metadata)?; + vortex_ensure!( + metadata.is_empty(), + "NullArray expects empty metadata, got {} bytes", + metadata.len() + ); Ok(NullData::new()) } diff --git a/vortex-array/src/arrays/patched/array.rs b/vortex-array/src/arrays/patched/array.rs index a9913b2c118..56f4ecb984b 100644 --- a/vortex-array/src/arrays/patched/array.rs +++ b/vortex-array/src/arrays/patched/array.rs @@ -252,31 +252,6 @@ impl PatchedArray { self.base_array().len() } - pub fn validate_against_outer( - &self, - dtype: &crate::dtype::DType, - len: usize, - ) -> VortexResult<()> { - vortex_ensure!( - self.base_array().dtype() == dtype, - "PatchedArray base dtype {} does not match outer dtype {}", - self.base_array().dtype(), - dtype - ); - vortex_ensure!( - self.len() == len, - "PatchedArray base len {} does not match outer len {}", - self.len(), - len - ); - vortex_ensure!( - self.patch_indices().len() == self.patch_values().len(), - "PatchedArray patch indices len {} does not match patch values len {}", - self.patch_indices().len(), - self.patch_values().len() - ); - Ok(()) - } } impl PatchedArray { diff --git a/vortex-array/src/arrays/patched/vtable/mod.rs b/vortex-array/src/arrays/patched/vtable/mod.rs index c772e94bb38..0e592452e27 100644 --- a/vortex-array/src/arrays/patched/vtable/mod.rs +++ b/vortex-array/src/arrays/patched/vtable/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use prost::Message; mod kernels; mod operations; mod slice; @@ -19,13 +20,10 @@ use crate::ArrayEq; use crate::ArrayHash; use crate::ArrayRef; use crate::Canonical; -use crate::DeserializeMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::IntoArray; use crate::Precision; -use crate::ProstMetadata; -use crate::SerializeMetadata; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; @@ -115,7 +113,25 @@ impl VTable for Patched { } fn validate(&self, data: &PatchedArray, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate_against_outer(dtype, len) + vortex_ensure!( + data.base_array().dtype() == dtype, + "PatchedArray base dtype {} does not match outer dtype {}", + data.base_array().dtype(), + dtype + ); + vortex_ensure!( + data.len() == len, + "PatchedArray base len {} does not match outer len {}", + data.len(), + len + ); + vortex_ensure!( + data.patch_indices().len() == data.patch_values().len(), + "PatchedArray patch indices len {} does not match patch values len {}", + data.patch_indices().len(), + data.patch_values().len() + ); + Ok(()) } fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { @@ -138,12 +154,12 @@ impl VTable for Patched { fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some( - ProstMetadata(PatchedMetadata { + PatchedMetadata { n_patches: u32::try_from(array.patch_indices().len())?, n_lanes: u32::try_from(array.n_lanes)?, offset: u32::try_from(array.offset)?, - }) - .serialize(), + } + .encode_to_vec(), )) } @@ -156,7 +172,7 @@ impl VTable for Patched { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - let metadata = ProstMetadata::::deserialize(metadata)?; + let metadata = PatchedMetadata::decode(metadata)?; let n_patches = metadata.n_patches as usize; let n_lanes = metadata.n_lanes as usize; let offset = metadata.offset as usize; diff --git a/vortex-array/src/arrays/primitive/vtable/mod.rs b/vortex-array/src/arrays/primitive/vtable/mod.rs index 4221a3d8237..ab29d6305f5 100644 --- a/vortex-array/src/arrays/primitive/vtable/mod.rs +++ b/vortex-array/src/arrays/primitive/vtable/mod.rs @@ -8,7 +8,6 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use crate::ArrayRef; -use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::array::Array; @@ -111,7 +110,12 @@ impl VTable for Primitive { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - ::deserialize(metadata)?; + if !metadata.is_empty() { + vortex_bail!( + "PrimitiveArray expects empty metadata, got {} bytes", + metadata.len() + ); + } if buffers.len() != 1 { vortex_bail!("Expected 1 buffer, got {}", buffers.len()); } diff --git a/vortex-array/src/arrays/struct_/array.rs b/vortex-array/src/arrays/struct_/array.rs index 032fc426f58..46628b84aab 100644 --- a/vortex-array/src/arrays/struct_/array.rs +++ b/vortex-array/src/arrays/struct_/array.rs @@ -547,28 +547,6 @@ impl StructData { None } - pub fn validate_against_outer(&self, dtype: &DType, len: usize) -> VortexResult<()> { - match dtype { - DType::Struct(_, _) => {} - _ => vortex_bail!("Expected struct dtype, found {:?}", dtype), - } - if self.len() != len { - vortex_bail!( - InvalidArgument: "StructArray length {} does not match outer length {}", - self.len(), - len - ); - } - let data_dtype = self.dtype(); - if &data_dtype != dtype { - vortex_bail!( - InvalidArgument: "StructArray dtype {} does not match outer dtype {}", - data_dtype, - dtype - ); - } - Ok(()) - } } impl Array { diff --git a/vortex-array/src/arrays/struct_/vtable/mod.rs b/vortex-array/src/arrays/struct_/vtable/mod.rs index 33fa42f91b3..56b263d91dc 100644 --- a/vortex-array/src/arrays/struct_/vtable/mod.rs +++ b/vortex-array/src/arrays/struct_/vtable/mod.rs @@ -10,7 +10,6 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; -use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::array::Array; @@ -67,7 +66,26 @@ impl VTable for Struct { } fn validate(&self, data: &StructData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate_against_outer(dtype, len) + match dtype { + DType::Struct(_, _) => {} + _ => vortex_bail!("Expected struct dtype, found {:?}", dtype), + } + if data.len() != len { + vortex_bail!( + InvalidArgument: "StructArray length {} does not match outer length {}", + data.len(), + len + ); + } + let data_dtype = data.dtype(); + if &data_dtype != dtype { + vortex_bail!( + InvalidArgument: "StructArray dtype {} does not match outer dtype {}", + data_dtype, + dtype + ); + } + Ok(()) } fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { @@ -92,7 +110,9 @@ impl VTable for Struct { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - ::deserialize(metadata)?; + if !metadata.is_empty() { + vortex_bail!("StructArray expects empty metadata, got {} bytes", metadata.len()); + } let DType::Struct(struct_dtype, nullability) = dtype else { vortex_bail!("Expected struct dtype, found {:?}", dtype) }; diff --git a/vortex-array/src/arrays/varbin/array.rs b/vortex-array/src/arrays/varbin/array.rs index 4fdb1bda6c4..a4b041eca84 100644 --- a/vortex-array/src/arrays/varbin/array.rs +++ b/vortex-array/src/arrays/varbin/array.rs @@ -496,21 +496,6 @@ impl VarBinData { (self.dtype(), self.bytes, offsets, validity) } - pub fn validate_against_outer(&self, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - self.len() == len, - "VarBinArray length {} does not match outer length {}", - self.len(), - len - ); - vortex_ensure!( - self.dtype() == *dtype, - "VarBinArray dtype {} does not match outer dtype {}", - self.dtype(), - dtype - ); - Ok(()) - } } impl Array { diff --git a/vortex-array/src/arrays/varbin/vtable/mod.rs b/vortex-array/src/arrays/varbin/vtable/mod.rs index 8e66b4cb12a..0b6813ddc99 100644 --- a/vortex-array/src/arrays/varbin/vtable/mod.rs +++ b/vortex-array/src/arrays/varbin/vtable/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use prost::Message; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -8,12 +9,9 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use crate::ArrayRef; -use crate::DeserializeMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::IntoArray; -use crate::ProstMetadata; -use crate::SerializeMetadata; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; @@ -77,7 +75,19 @@ impl VTable for VarBin { } fn validate(&self, data: &VarBinData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate_against_outer(dtype, len) + vortex_ensure!( + data.len() == len, + "VarBinArray length {} does not match outer length {}", + data.len(), + len + ); + vortex_ensure!( + data.dtype() == *dtype, + "VarBinArray dtype {} does not match outer dtype {}", + data.dtype(), + dtype + ); + Ok(()) } fn buffer(array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { @@ -96,11 +106,11 @@ impl VTable for VarBin { fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some( - ProstMetadata(VarBinMetadata { + VarBinMetadata { offsets_ptype: PType::try_from(array.offsets().dtype()) .vortex_expect("Must be a valid PType") as i32, - }) - .serialize(), + } + .encode_to_vec(), )) } @@ -114,7 +124,7 @@ impl VTable for VarBin { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - let metadata = ProstMetadata::::deserialize(metadata)?; + let metadata = VarBinMetadata::decode(metadata)?; let validity = if children.len() == 1 { Validity::from(dtype.nullability()) } else if children.len() == 2 { diff --git a/vortex-array/src/arrays/varbinview/array.rs b/vortex-array/src/arrays/varbinview/array.rs index 52e58c72572..1fd82e5f9f1 100644 --- a/vortex-array/src/arrays/varbinview/array.rs +++ b/vortex-array/src/arrays/varbinview/array.rs @@ -416,22 +416,6 @@ impl VarBinViewData { } } - pub fn validate_against_outer(&self, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - self.len() == len, - "VarBinViewArray length {} does not match outer length {}", - self.len(), - len - ); - vortex_ensure!( - self.dtype() == *dtype, - "VarBinViewArray dtype {} does not match outer dtype {}", - self.dtype(), - dtype - ); - Ok(()) - } - /// Access to the primitive views buffer. /// /// Variable-sized binary view buffer contain a "view" child array, with 16-byte entries that diff --git a/vortex-array/src/arrays/varbinview/vtable/mod.rs b/vortex-array/src/arrays/varbinview/vtable/mod.rs index fa397991280..7c2b674aa9e 100644 --- a/vortex-array/src/arrays/varbinview/vtable/mod.rs +++ b/vortex-array/src/arrays/varbinview/vtable/mod.rs @@ -14,7 +14,6 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; -use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; @@ -84,7 +83,19 @@ impl VTable for VarBinView { } fn validate(&self, data: &VarBinViewData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate_against_outer(dtype, len) + vortex_ensure!( + data.len() == len, + "VarBinViewArray length {} does not match outer length {}", + data.len(), + len + ); + vortex_ensure!( + data.dtype() == *dtype, + "VarBinViewArray dtype {} does not match outer dtype {}", + data.dtype(), + dtype + ); + Ok(()) } fn buffer(array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { @@ -123,7 +134,12 @@ impl VTable for VarBinView { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult { - ::deserialize(metadata)?; + if !metadata.is_empty() { + vortex_bail!( + "VarBinViewArray expects empty metadata, got {} bytes", + metadata.len() + ); + } let Some((views_handle, data_handles)) = buffers.split_last() else { vortex_bail!("Expected at least 1 buffer, got 0"); }; diff --git a/vortex-array/src/arrays/variant/vtable/mod.rs b/vortex-array/src/arrays/variant/vtable/mod.rs index 497a889fbc2..49981a6f955 100644 --- a/vortex-array/src/arrays/variant/vtable/mod.rs +++ b/vortex-array/src/arrays/variant/vtable/mod.rs @@ -13,7 +13,6 @@ use vortex_error::vortex_panic; use crate::ArrayEq; use crate::ArrayHash; use crate::ArrayRef; -use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; @@ -103,7 +102,11 @@ impl VTable for Variant { children: &dyn ArrayChildren, _session: &vortex_session::VortexSession, ) -> VortexResult { - ::deserialize(metadata)?; + vortex_ensure!( + metadata.is_empty(), + "VariantArray expects empty metadata, got {} bytes", + metadata.len() + ); vortex_ensure!(matches!(dtype, DType::Variant(_)), "Expected Variant DType"); vortex_ensure!( children.len() == 1, diff --git a/vortex-array/src/test_harness.rs b/vortex-array/src/test_harness.rs index 348f79829fd..cee718f3920 100644 --- a/vortex-array/src/test_harness.rs +++ b/vortex-array/src/test_harness.rs @@ -8,25 +8,18 @@ use goldenfile::differs::binary_diff; use itertools::Itertools; use vortex_error::VortexResult; -use crate::DeserializeMetadata; -use crate::SerializeMetadata; use crate::arrays::BoolArray; /// Check that a named metadata matches its previous versioning. /// /// Goldenfile takes care of checking for equality against a checked-in file. #[allow(clippy::unwrap_used)] -pub fn check_metadata(name: &str, metadata: T) -where - T: SerializeMetadata, - T: DeserializeMetadata, -{ +pub fn check_metadata(name: &str, metadata: &[u8]) { let mut mint = Mint::new("goldenfiles/"); - let meta = metadata.serialize(); let mut f = mint .new_goldenfile_with_differ(name, Box::new(binary_diff)) .unwrap(); - f.write_all(&meta).unwrap(); + f.write_all(metadata).unwrap(); } /// Outputs the indices of the true values in a BoolArray From 8e6d4e27330dac83806a61eca83d89324acad83c Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Fri, 3 Apr 2026 10:54:49 -0400 Subject: [PATCH 07/15] Advanced frame parsing Signed-off-by: Nicholas Gates --- encodings/alp/src/alp/array.rs | 4 +- encodings/alp/src/alp_rd/array.rs | 2 +- encodings/bytebool/src/array.rs | 5 ++- encodings/bytebool/src/compute.rs | 5 ++- .../fastlanes/benches/bitpacking_take.rs | 12 +++--- encodings/fastlanes/src/bitpacking/mod.rs | 2 +- .../fastlanes/src/bitpacking/vtable/mod.rs | 2 +- encodings/parquet-variant/src/vtable.rs | 3 -- encodings/runend/src/arbitrary.rs | 9 +++-- encodings/runend/src/array.rs | 4 +- encodings/runend/src/arrow.rs | 3 +- encodings/sequence/src/array.rs | 6 +-- encodings/sequence/src/compute/min_max.rs | 3 +- encodings/zigzag/src/array.rs | 5 ++- encodings/zstd/benches/listview_rebuild.rs | 6 ++- encodings/zstd/src/array.rs | 1 - fuzz/src/array/filter.rs | 2 +- fuzz/src/array/mask.rs | 2 +- fuzz/src/array/slice.rs | 2 +- vortex-array/public-api.lock | 10 +++++ vortex-array/src/array/typed.rs | 10 ++++- vortex-array/src/array/vtable/dyn_.rs | 6 ++- vortex-array/src/arrays/bool/array.rs | 1 - vortex-array/src/arrays/bool/mod.rs | 2 +- vortex-array/src/arrays/bool/vtable/mod.rs | 2 +- vortex-array/src/arrays/chunked/array.rs | 2 +- vortex-array/src/arrays/chunked/vtable/mod.rs | 9 +++-- vortex-array/src/arrays/decimal/mod.rs | 2 +- vortex-array/src/arrays/decimal/vtable/mod.rs | 2 +- vortex-array/src/arrays/dict/array.rs | 1 + vortex-array/src/arrays/dict/vtable/mod.rs | 4 +- .../src/arrays/filter/execute/struct_.rs | 9 +---- .../src/arrays/listview/conversion.rs | 2 +- vortex-array/src/arrays/listview/mod.rs | 2 +- vortex-array/src/arrays/masked/execute.rs | 2 +- vortex-array/src/arrays/masked/vtable/mod.rs | 5 ++- vortex-array/src/arrays/null/mod.rs | 6 +++ vortex-array/src/arrays/patched/array.rs | 4 ++ .../src/arrays/primitive/array/cast.rs | 14 +++++++ vortex-array/src/arrays/primitive/mod.rs | 2 +- vortex-array/src/arrays/struct_/array.rs | 21 +++++----- .../src/arrays/struct_/compute/mask.rs | 2 +- .../src/arrays/struct_/compute/slice.rs | 2 +- .../src/arrays/struct_/compute/take.rs | 4 +- vortex-array/src/arrays/struct_/mod.rs | 2 +- vortex-array/src/arrays/struct_/vtable/mod.rs | 7 +++- vortex-array/src/arrays/varbin/array.rs | 39 +++++++++---------- vortex-array/src/arrays/varbinview/array.rs | 30 +++++++------- vortex-array/src/arrays/varbinview/mod.rs | 2 +- vortex-array/src/arrays/variant/mod.rs | 11 ++++-- .../src/display/extractors/metadata.rs | 1 + vortex-array/src/scalar_fn/fns/select.rs | 2 +- vortex-btrblocks/src/schemes/rle.rs | 5 ++- vortex-cuda/benches/date_time_parts_cuda.rs | 9 ++--- vortex-cuda/benches/dynamic_dispatch_cuda.rs | 13 ++++--- vortex-cuda/benches/for_cuda.rs | 15 ++----- vortex-cuda/src/kernel/arrays/dict.rs | 12 +----- vortex-cuda/src/kernel/encodings/bitpacked.rs | 4 +- .../kernel/encodings/decimal_byte_parts.rs | 2 +- vortex-cuda/src/kernel/encodings/zstd.rs | 8 +--- vortex-python/src/arrays/py/array.rs | 10 +++-- vortex/benches/single_encoding_throughput.rs | 29 +++++++++----- 62 files changed, 224 insertions(+), 181 deletions(-) diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index 322d67e6835..38d20f7c5c9 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -417,9 +417,9 @@ impl ALPData { Ok(Self { slots, - exponents, patch_offset, patch_offset_within_chunk, + exponents, }) } @@ -440,9 +440,9 @@ impl ALPData { Self { slots, - exponents, patch_offset, patch_offset_within_chunk, + exponents, } } } diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index 6922e40733e..811b082ef5c 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -529,9 +529,9 @@ impl ALPRDData { Ok(Self { slots, + left_parts_patches, left_parts_dictionary, right_bit_width, - left_parts_patches, }) } diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index b9bf9aa3a5f..0449765b3a6 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -96,7 +96,10 @@ impl VTable for ByteBool { _session: &VortexSession, ) -> VortexResult { if !metadata.is_empty() { - vortex_bail!("ByteBoolArray expects empty metadata, got {} bytes", metadata.len()); + vortex_bail!( + "ByteBoolArray expects empty metadata, got {} bytes", + metadata.len() + ); } let validity = if children.is_empty() { Validity::from(dtype.nullability()) diff --git a/encodings/bytebool/src/compute.rs b/encodings/bytebool/src/compute.rs index d929ed7b2c8..904c8e5cf93 100644 --- a/encodings/bytebool/src/compute.rs +++ b/encodings/bytebool/src/compute.rs @@ -84,8 +84,6 @@ impl TakeExecute for ByteBool { #[cfg(test)] mod tests { - use super::*; - use crate::ByteBoolArray; use rstest::rstest; use vortex_array::assert_arrays_eq; use vortex_array::builtins::ArrayBuiltins; @@ -98,6 +96,9 @@ mod tests { use vortex_array::dtype::Nullability; use vortex_array::scalar_fn::fns::operators::Operator; + use super::*; + use crate::ByteBoolArray; + fn bb(v: Vec) -> ByteBoolArray { ByteBool::from_vec(v, Validity::AllValid) } diff --git a/encodings/fastlanes/benches/bitpacking_take.rs b/encodings/fastlanes/benches/bitpacking_take.rs index e0325b7a481..6129f479d47 100644 --- a/encodings/fastlanes/benches/bitpacking_take.rs +++ b/encodings/fastlanes/benches/bitpacking_take.rs @@ -160,9 +160,9 @@ fn patched_take_10_stratified(bencher: Bencher) { let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); - assert!(packed.patches().is_some()); + assert!(packed.patches(packed.len()).is_some()); assert_eq!( - packed.patches().unwrap().num_patches(), + packed.patches(packed.len()).unwrap().num_patches(), NUM_EXCEPTIONS as usize ); @@ -185,9 +185,9 @@ fn patched_take_10_contiguous(bencher: Bencher) { let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); - assert!(packed.patches().is_some()); + assert!(packed.patches(packed.len()).is_some()); assert_eq!( - packed.patches().unwrap().num_patches(), + packed.patches(packed.len()).unwrap().num_patches(), NUM_EXCEPTIONS as usize ); @@ -249,9 +249,9 @@ fn patched_take_10k_contiguous_patches(bencher: Bencher) { let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); - assert!(packed.patches().is_some()); + assert!(packed.patches(packed.len()).is_some()); assert_eq!( - packed.patches().unwrap().num_patches(), + packed.patches(packed.len()).unwrap().num_patches(), NUM_EXCEPTIONS as usize ); diff --git a/encodings/fastlanes/src/bitpacking/mod.rs b/encodings/fastlanes/src/bitpacking/mod.rs index 0e6e36804ea..d777d372773 100644 --- a/encodings/fastlanes/src/bitpacking/mod.rs +++ b/encodings/fastlanes/src/bitpacking/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::BitPackedDataParts; pub use array::BitPackedData; +pub use array::BitPackedDataParts; pub use array::bitpack_compress; pub use array::bitpack_decompress; pub use array::unpack_iter; diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs index ecebaa01183..4ac2d748f48 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs @@ -37,8 +37,8 @@ use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; -use crate::BitPackedDataParts; use crate::BitPackedData; +use crate::BitPackedDataParts; use crate::bitpack_decompress::unpack_array; use crate::bitpack_decompress::unpack_into_primitive_builder; use crate::bitpacking::array::NUM_SLOTS; diff --git a/encodings/parquet-variant/src/vtable.rs b/encodings/parquet-variant/src/vtable.rs index 0bb166f12ba..083cfc42261 100644 --- a/encodings/parquet-variant/src/vtable.rs +++ b/encodings/parquet-variant/src/vtable.rs @@ -208,7 +208,6 @@ impl VTable for ParquetVariant { let typed_value = if has_typed_value { // typed_value can be any type — primitive, list, struct, etc. let dtype = typed_value_dtype - .clone() .ok_or_else(|| vortex_err!("typed_value_dtype missing for typed_value child"))?; let tv = children.get(child_idx, &dtype, len)?; Some(tv) @@ -274,8 +273,6 @@ mod tests { use vortex_session::registry::ReadContext; use crate::ParquetVariant; - use crate::ParquetVariantData; - fn roundtrip(array: ArrayRef) -> ArrayRef { let dtype = array.dtype().clone(); let len = array.len(); diff --git a/encodings/runend/src/arbitrary.rs b/encodings/runend/src/arbitrary.rs index a63fa9bb2ed..5347935c578 100644 --- a/encodings/runend/src/arbitrary.rs +++ b/encodings/runend/src/arbitrary.rs @@ -14,11 +14,12 @@ use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexExpect; -use crate::RunEndData; +use crate::RunEnd; +use crate::RunEndArray; /// A wrapper type to implement `Arbitrary` for `RunEndArray`. #[derive(Clone, Debug)] -pub struct ArbitraryRunEndArray(pub RunEndData); +pub struct ArbitraryRunEndArray(pub RunEndArray); impl<'a> Arbitrary<'a> for ArbitraryRunEndArray { fn arbitrary(u: &mut Unstructured<'a>) -> Result { @@ -42,7 +43,7 @@ impl ArbitraryRunEndArray { // Empty RunEndArray let ends = PrimitiveArray::from_iter(Vec::::new()).into_array(); let values = ArbitraryArray::arbitrary_with(u, Some(0), dtype)?.0; - let runend_array = RunEndData::try_new(ends, values) + let runend_array = RunEnd::try_new(ends, values) .vortex_expect("Empty RunEndArray creation should succeed"); return Ok(ArbitraryRunEndArray(runend_array)); } @@ -54,7 +55,7 @@ impl ArbitraryRunEndArray { // Each end must be > previous end, and first end must be >= 1 let ends = random_strictly_sorted_ends(u, num_runs, len)?; - let runend_array = RunEndData::try_new(ends, values) + let runend_array = RunEnd::try_new(ends, values) .vortex_expect("RunEndArray creation should succeed in arbitrary impl"); Ok(ArbitraryRunEndArray(runend_array)) diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index 35c1b560626..01f266a933f 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -319,9 +319,7 @@ impl RunEndData { if offset != 0 && length != 0 { let first_run_end = usize::try_from(&ends.scalar_at(0)?)?; if first_run_end < offset { - vortex_bail!( - "First run end {first_run_end} must be >= offset {offset}" - ); + vortex_bail!("First run end {first_run_end} must be >= offset {offset}"); } } diff --git a/encodings/runend/src/arrow.rs b/encodings/runend/src/arrow.rs index e4fbd26f5f6..8d9e674f778 100644 --- a/encodings/runend/src/arrow.rs +++ b/encodings/runend/src/arrow.rs @@ -298,8 +298,7 @@ mod tests { #[case] expected_ends: &[i32], #[case] expected_values: &[i32], ) -> VortexResult<()> { - let array = - RunEnd::encode(PrimitiveArray::from_iter(input.iter().copied()).into_array())?; + let array = RunEnd::encode(PrimitiveArray::from_iter(input.iter().copied()).into_array())?; let sliced = array.into_array().slice(slice_range.clone())?; let target = ree_type(DataType::Int32, DataType::Int32); let result = execute(sliced, &target)?; diff --git a/encodings/sequence/src/array.rs b/encodings/sequence/src/array.rs index 42d69821700..8ad97e91f76 100644 --- a/encodings/sequence/src/array.rs +++ b/encodings/sequence/src/array.rs @@ -203,7 +203,6 @@ impl SequenceData { PValue::from(value) }) } - } impl VTable for Sequence { @@ -381,7 +380,7 @@ impl Sequence { ); // SAFETY: we don't have duplicate stats. - let stats_set = unsafe { + unsafe { StatsSet::new_unchecked(vec![ (Stat::IsSorted, StatPrecision::Exact(is_sorted.into())), ( @@ -389,8 +388,7 @@ impl Sequence { StatPrecision::Exact(is_strict_sorted.into()), ), ]) - }; - stats_set + } } /// Construct a new [`SequenceArray`] from pre-validated parts. diff --git a/encodings/sequence/src/compute/min_max.rs b/encodings/sequence/src/compute/min_max.rs index 67a7a4e6ebb..2ff8b2abd58 100644 --- a/encodings/sequence/src/compute/min_max.rs +++ b/encodings/sequence/src/compute/min_max.rs @@ -48,8 +48,7 @@ impl DynAggregateKernel for SequenceMinMaxKernel { } let base = seq.base(); - let last = SequenceData::try_last(base, seq.multiplier(), seq.ptype(), seq.len()) - .expect("validated array"); + let last = SequenceData::try_last(base, seq.multiplier(), seq.ptype(), seq.len())?; // Determine min and max based on multiplier direction. // For unsigned types, multiplier is always >= 0. diff --git a/encodings/zigzag/src/array.rs b/encodings/zigzag/src/array.rs index 29122577e6b..df8b0195d72 100644 --- a/encodings/zigzag/src/array.rs +++ b/encodings/zigzag/src/array.rs @@ -96,7 +96,10 @@ impl VTable for ZigZag { _session: &VortexSession, ) -> VortexResult { if !metadata.is_empty() { - vortex_bail!("ZigZagArray expects empty metadata, got {} bytes", metadata.len()); + vortex_bail!( + "ZigZagArray expects empty metadata, got {} bytes", + metadata.len() + ); } if children.len() != 1 { vortex_bail!("Expected 1 child, got {}", children.len()); diff --git a/encodings/zstd/benches/listview_rebuild.rs b/encodings/zstd/benches/listview_rebuild.rs index 0e91342c8ac..55690cdccae 100644 --- a/encodings/zstd/benches/listview_rebuild.rs +++ b/encodings/zstd/benches/listview_rebuild.rs @@ -10,13 +10,17 @@ use vortex_array::arrays::VarBinViewArray; use vortex_array::arrays::listview::ListViewRebuildMode; use vortex_array::validity::Validity; use vortex_buffer::Buffer; +use vortex_zstd::Zstd; use vortex_zstd::ZstdData; #[divan::bench(sample_size = 1000)] fn rebuild_naive(bencher: Bencher) { let dudes = VarBinViewArray::from_iter_str(["Washington", "Adams", "Jefferson", "Madison"]) .into_array(); - let dudes = ZstdData::from_array(dudes, 9, 1024).unwrap().into_array(); + let dtype = dudes.dtype().clone(); + let dudes = Zstd::try_new(dtype, ZstdData::from_array(dudes, 9, 1024).unwrap()) + .unwrap() + .into_array(); let offsets = std::iter::repeat_n(0u32, 1024) .collect::>() diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index 46184ae8e2f..6c4929aa40e 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -307,7 +307,6 @@ impl Zstd { pub fn decompress(array: &ZstdArray, ctx: &mut ExecutionCtx) -> VortexResult { array.data().decompress(array.dtype(), ctx) } - } /// The validity bitmap indicating which elements are non-null. diff --git a/fuzz/src/array/filter.rs b/fuzz/src/array/filter.rs index 7db3e1f8f74..9431cac68f6 100644 --- a/fuzz/src/array/filter.rs +++ b/fuzz/src/array/filter.rs @@ -98,7 +98,7 @@ pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult StructArray::try_new_with_dtype( filtered_children, - struct_array.struct_fields().clone(), + struct_array.struct_fields(), filter.iter().filter(|b| **b).map(|b| *b as usize).sum(), validity, ) diff --git a/fuzz/src/array/mask.rs b/fuzz/src/array/mask.rs index 02974d72df7..89192b3e360 100644 --- a/fuzz/src/array/mask.rs +++ b/fuzz/src/array/mask.rs @@ -117,7 +117,7 @@ pub fn mask_canonical_array(canonical: Canonical, mask: &Mask) -> VortexResult>>()?; StructArray::try_new_with_dtype( sliced_children, - struct_array.struct_fields().clone(), + struct_array.struct_fields(), stop - start, validity, ) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 1ef22101ebe..697653afd24 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -2916,6 +2916,10 @@ impl core::clone::Clone for vortex_array::arrays::null::NullData pub fn vortex_array::arrays::null::NullData::clone(&self) -> vortex_array::arrays::null::NullData +impl core::default::Default for vortex_array::arrays::null::NullData + +pub fn vortex_array::arrays::null::NullData::default() -> Self + impl core::fmt::Debug for vortex_array::arrays::null::NullData pub fn vortex_array::arrays::null::NullData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result @@ -3014,6 +3018,8 @@ impl vortex_array::arrays::patched::PatchedArray pub fn vortex_array::arrays::patched::PatchedArray::base_array(&self) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::PatchedArray::is_empty(&self) -> bool + pub fn vortex_array::arrays::patched::PatchedArray::lane_offsets(&self) -> &vortex_array::ArrayRef pub fn vortex_array::arrays::patched::PatchedArray::len(&self) -> usize @@ -5892,6 +5898,8 @@ impl vortex_array::arrays::patched::PatchedArray pub fn vortex_array::arrays::patched::PatchedArray::base_array(&self) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::PatchedArray::is_empty(&self) -> bool + pub fn vortex_array::arrays::patched::PatchedArray::lane_offsets(&self) -> &vortex_array::ArrayRef pub fn vortex_array::arrays::patched::PatchedArray::len(&self) -> usize @@ -21622,6 +21630,8 @@ impl vortex_array::Array pub fn vortex_array::Array::narrow(&self) -> vortex_error::VortexResult +pub fn vortex_array::Array::reinterpret_cast(&self, ptype: vortex_array::dtype::PType) -> Self + impl vortex_array::Array pub fn vortex_array::Array::patch(self, patches: &vortex_array::patches::Patches, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult diff --git a/vortex-array/src/array/typed.rs b/vortex-array/src/array/typed.rs index ae846b71265..aadd2485c94 100644 --- a/vortex-array/src/array/typed.rs +++ b/vortex-array/src/array/typed.rs @@ -180,7 +180,13 @@ impl Array { /// Caller must ensure the provided parts are logically consistent. pub(crate) unsafe fn from_parts_unchecked(new: ArrayParts) -> Self { let inner = ArrayRef::from_inner(Arc::new(unsafe { - ArrayInner::::from_data_unchecked(new.vtable, new.dtype, new.len, new.data, ArrayStats::default()) + ArrayInner::::from_data_unchecked( + new.vtable, + new.dtype, + new.len, + new.data, + ArrayStats::default(), + ) })); Self { inner, @@ -443,9 +449,9 @@ mod tests { use vortex_buffer::buffer; use super::Array; - use crate::assert_arrays_eq; use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; + use crate::assert_arrays_eq; use crate::validity::Validity; #[test] diff --git a/vortex-array/src/array/vtable/dyn_.rs b/vortex-array/src/array/vtable/dyn_.rs index 63df8c2f880..681267403e2 100644 --- a/vortex-array/src/array/vtable/dyn_.rs +++ b/vortex-array/src/array/vtable/dyn_.rs @@ -84,8 +84,10 @@ impl DynVTable for V { session: &VortexSession, ) -> VortexResult { let inner = self.deserialize(dtype, len, metadata, buffers, children, session)?; - Ok(Array::::try_from_parts(ArrayParts::new(self.clone(), dtype.clone(), len, inner))? - .into_array()) + Ok( + Array::::try_from_parts(ArrayParts::new(self.clone(), dtype.clone(), len, inner))? + .into_array(), + ) } fn with_slots(&self, array: ArrayRef, slots: Vec>) -> VortexResult { diff --git a/vortex-array/src/arrays/bool/array.rs b/vortex-array/src/arrays/bool/array.rs index ec06e4d1106..29eb9f1d0f2 100644 --- a/vortex-array/src/arrays/bool/array.rs +++ b/vortex-array/src/arrays/bool/array.rs @@ -241,7 +241,6 @@ impl Array { pub fn into_bit_buffer(self) -> BitBuffer { self.into_data().into_bit_buffer() } - } /// Internal constructors on BoolData (used by Array constructors and VTable::build). diff --git a/vortex-array/src/arrays/bool/mod.rs b/vortex-array/src/arrays/bool/mod.rs index 0a825ae72fa..b0f20794ca8 100644 --- a/vortex-array/src/arrays/bool/mod.rs +++ b/vortex-array/src/arrays/bool/mod.rs @@ -4,8 +4,8 @@ mod array; mod patch; -pub use array::BoolDataParts; pub use array::BoolData; +pub use array::BoolDataParts; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/bool/vtable/mod.rs b/vortex-array/src/arrays/bool/vtable/mod.rs index 0842a90a398..bf9d13712e8 100644 --- a/vortex-array/src/arrays/bool/vtable/mod.rs +++ b/vortex-array/src/arrays/bool/vtable/mod.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use prost::Message; use kernel::PARENT_KERNELS; +use prost::Message; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; diff --git a/vortex-array/src/arrays/chunked/array.rs b/vortex-array/src/arrays/chunked/array.rs index 857eb43c777..f3b3a9fc24c 100644 --- a/vortex-array/src/arrays/chunked/array.rs +++ b/vortex-array/src/arrays/chunked/array.rs @@ -50,7 +50,7 @@ impl ChunkedData { slots.push(Some( Array::try_from_parts(ArrayParts::new( crate::arrays::Primitive, - chunk_offsets.dtype().clone(), + chunk_offsets.dtype(), chunk_offsets.len(), chunk_offsets.clone(), )) diff --git a/vortex-array/src/arrays/chunked/vtable/mod.rs b/vortex-array/src/arrays/chunked/vtable/mod.rs index 93b5c96a600..6349bfa97ec 100644 --- a/vortex-array/src/arrays/chunked/vtable/mod.rs +++ b/vortex-array/src/arrays/chunked/vtable/mod.rs @@ -135,7 +135,10 @@ impl VTable for Chunked { _session: &VortexSession, ) -> VortexResult { if !metadata.is_empty() { - vortex_bail!("ChunkedArray expects empty metadata, got {} bytes", metadata.len()); + vortex_bail!( + "ChunkedArray expects empty metadata, got {} bytes", + metadata.len() + ); } if children.is_empty() { vortex_bail!("Chunked array needs at least one child"); @@ -167,7 +170,7 @@ impl VTable for Chunked { }) .try_collect()?; - let chunk_offsets = PrimitiveData::new(chunk_offsets_buf.clone(), Validity::NonNullable); + let chunk_offsets = PrimitiveData::new(chunk_offsets_buf, Validity::NonNullable); let slots = ChunkedData::make_slots(&chunk_offsets, &chunks); // Construct directly using the struct fields to avoid recomputing chunk_offsets @@ -214,7 +217,7 @@ impl VTable for Chunked { .ok_or_else(|| vortex_err!("chunk slot must not be None")) }) .try_collect()?; - array.chunk_offsets = PrimitiveData::new(chunk_offsets_buf.clone(), Validity::NonNullable); + array.chunk_offsets = PrimitiveData::new(chunk_offsets_buf, Validity::NonNullable); array.empty_dtype = chunks.is_empty().then_some(array.dtype().clone()); array.chunks = chunks; array.slots = slots; diff --git a/vortex-array/src/arrays/decimal/mod.rs b/vortex-array/src/arrays/decimal/mod.rs index fa52017d829..a0b90613f52 100644 --- a/vortex-array/src/arrays/decimal/mod.rs +++ b/vortex-array/src/arrays/decimal/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::DecimalDataParts; pub use array::DecimalData; +pub use array::DecimalDataParts; pub use vtable::DecimalArray; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/decimal/vtable/mod.rs b/vortex-array/src/arrays/decimal/vtable/mod.rs index fba5962b82c..dc7a46c779d 100644 --- a/vortex-array/src/arrays/decimal/vtable/mod.rs +++ b/vortex-array/src/arrays/decimal/vtable/mod.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use prost::Message; use kernel::PARENT_KERNELS; +use prost::Message; use vortex_buffer::Alignment; use vortex_error::VortexResult; use vortex_error::vortex_bail; diff --git a/vortex-array/src/arrays/dict/array.rs b/vortex-array/src/arrays/dict/array.rs index a347862c601..c452983f62c 100644 --- a/vortex-array/src/arrays/dict/array.rs +++ b/vortex-array/src/arrays/dict/array.rs @@ -449,6 +449,7 @@ mod test { #[test] fn test_dict_metadata() { use prost::Message; + use super::DictMetadata; use crate::test_harness::check_metadata; diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index fa4f243b091..25be2053332 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use prost::Message; use kernel::PARENT_KERNELS; +use prost::Message; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -11,8 +11,8 @@ use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; -use super::DictDataParts; use super::DictData; +use super::DictDataParts; use super::DictMetadata; use super::array::NUM_SLOTS; use super::array::SLOT_NAMES; diff --git a/vortex-array/src/arrays/filter/execute/struct_.rs b/vortex-array/src/arrays/filter/execute/struct_.rs index 74540a8739d..e424dbd2d33 100644 --- a/vortex-array/src/arrays/filter/execute/struct_.rs +++ b/vortex-array/src/arrays/filter/execute/struct_.rs @@ -29,13 +29,8 @@ pub fn filter_struct(array: &StructArray, mask: &Arc) -> StructArray .map(|a| a.len()) .unwrap_or_else(|| mask.true_count()); - StructArray::try_new_with_dtype( - fields, - array.struct_fields().clone(), - length, - filtered_validity, - ) - .vortex_expect("filtered StructArray fields have consistent lengths") + StructArray::try_new_with_dtype(fields, array.struct_fields(), length, filtered_validity) + .vortex_expect("filtered StructArray fields have consistent lengths") } #[cfg(test)] diff --git a/vortex-array/src/arrays/listview/conversion.rs b/vortex-array/src/arrays/listview/conversion.rs index a24b541c1d8..550c745d499 100644 --- a/vortex-array/src/arrays/listview/conversion.rs +++ b/vortex-array/src/arrays/listview/conversion.rs @@ -39,7 +39,7 @@ pub fn list_view_from_list(list: ListArray, ctx: &mut ExecutionCtx) -> VortexRes // function might not expect the output `ListViewArray` to have a bunch of leading and trailing // garbage data when they turn it back into a `ListArray`. let data = list.reset_offsets(false).vortex_expect("This can't fail"); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); let list: ListArray = Array::try_from_parts(ArrayParts::new(List, dtype, len, data))?; diff --git a/vortex-array/src/arrays/listview/mod.rs b/vortex-array/src/arrays/listview/mod.rs index aa1a1159ef5..20dd6fe15d5 100644 --- a/vortex-array/src/arrays/listview/mod.rs +++ b/vortex-array/src/arrays/listview/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::ListViewDataParts; pub use array::ListViewData; +pub use array::ListViewDataParts; pub use vtable::ListViewArray; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/masked/execute.rs b/vortex-array/src/arrays/masked/execute.rs index bbf5b7fcb87..0f11a537ba2 100644 --- a/vortex-array/src/arrays/masked/execute.rs +++ b/vortex-array/src/arrays/masked/execute.rs @@ -177,7 +177,7 @@ fn mask_validity_struct( let len = array.len(); let new_validity = combine_validity(&array.validity(), mask, len, ctx)?; let fields = array.unmasked_fields(); - let struct_fields = array.struct_fields().clone(); + let struct_fields = array.struct_fields(); // SAFETY: We're only changing validity, not the data structure Ok(unsafe { StructArray::new_unchecked(fields, struct_fields, len, new_validity) }) } diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index 3e7b16f48c7..4a79d53b78b 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -102,7 +102,10 @@ impl VTable for Masked { _session: &VortexSession, ) -> VortexResult { if !metadata.is_empty() { - vortex_bail!("MaskedArray expects empty metadata, got {} bytes", metadata.len()); + vortex_bail!( + "MaskedArray expects empty metadata, got {} bytes", + metadata.len() + ); } if !buffers.is_empty() { vortex_bail!("Expected 0 buffer, got {}", buffers.len()); diff --git a/vortex-array/src/arrays/null/mod.rs b/vortex-array/src/arrays/null/mod.rs index 3319305a020..292a7c8ab4f 100644 --- a/vortex-array/src/arrays/null/mod.rs +++ b/vortex-array/src/arrays/null/mod.rs @@ -167,6 +167,12 @@ impl Array { } } +impl Default for NullData { + fn default() -> Self { + Self::new() + } +} + impl NullData { pub fn new() -> Self { Self { slots: vec![] } diff --git a/vortex-array/src/arrays/patched/array.rs b/vortex-array/src/arrays/patched/array.rs index 56f4ecb984b..f5470aac6df 100644 --- a/vortex-array/src/arrays/patched/array.rs +++ b/vortex-array/src/arrays/patched/array.rs @@ -252,6 +252,10 @@ impl PatchedArray { self.base_array().len() } + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } } impl PatchedArray { diff --git a/vortex-array/src/arrays/primitive/array/cast.rs b/vortex-array/src/arrays/primitive/array/cast.rs index c92977616ff..f213f6a60c5 100644 --- a/vortex-array/src/arrays/primitive/array/cast.rs +++ b/vortex-array/src/arrays/primitive/array/cast.rs @@ -61,6 +61,20 @@ impl PrimitiveData { } impl PrimitiveArray { + pub fn reinterpret_cast(&self, ptype: PType) -> Self { + if self.ptype() == ptype { + return self.clone(); + } + + assert_eq!( + self.ptype().byte_width(), + ptype.byte_width(), + "can't reinterpret cast between integers of two different widths" + ); + + PrimitiveArray::from_buffer_handle(self.buffer_handle().clone(), ptype, self.validity()) + } + /// Narrow the array to the smallest possible integer type that can represent all values. pub fn narrow(&self) -> VortexResult { if !self.ptype().is_int() { diff --git a/vortex-array/src/arrays/primitive/mod.rs b/vortex-array/src/arrays/primitive/mod.rs index e247e22c6d4..7c526c2cfbb 100644 --- a/vortex-array/src/arrays/primitive/mod.rs +++ b/vortex-array/src/arrays/primitive/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::PrimitiveDataParts; pub use array::PrimitiveData; +pub use array::PrimitiveDataParts; pub use array::chunk_range; pub use array::patch_chunk; pub use vtable::PrimitiveArray; diff --git a/vortex-array/src/arrays/struct_/array.rs b/vortex-array/src/arrays/struct_/array.rs index 46628b84aab..3758d0d75c6 100644 --- a/vortex-array/src/arrays/struct_/array.rs +++ b/vortex-array/src/arrays/struct_/array.rs @@ -512,7 +512,7 @@ impl StructData { pub fn remove_column(&mut self, name: impl Into) -> Option { let name = name.into(); - let struct_dtype = self.struct_fields().clone(); + let struct_dtype = self.struct_fields(); let len = self.len(); let position = struct_dtype @@ -546,7 +546,6 @@ impl StructData { } None } - } impl Array { @@ -558,7 +557,7 @@ impl Array { validity: Validity, ) -> Self { let data = StructData::new(names, fields, length, validity); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) .vortex_expect("StructData is always valid") @@ -572,7 +571,7 @@ impl Array { validity: Validity, ) -> VortexResult { let data = StructData::try_new(names, fields, length, validity)?; - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) } @@ -589,7 +588,7 @@ impl Array { validity: Validity, ) -> Self { let data = unsafe { StructData::new_unchecked(fields, dtype, length, validity) }; - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) .vortex_expect("StructData is always valid") @@ -603,7 +602,7 @@ impl Array { validity: Validity, ) -> VortexResult { let data = StructData::try_new_with_dtype(fields, dtype, length, validity)?; - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) } @@ -611,7 +610,7 @@ impl Array { /// Construct a `StructArray` from named fields. pub fn from_fields>(items: &[(N, ArrayRef)]) -> VortexResult { let data = StructData::from_fields(items)?; - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) } @@ -626,7 +625,7 @@ impl Array { validity: Validity, ) -> VortexResult { let data = StructData::try_from_iter_with_validity(iter, validity)?; - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) } @@ -636,7 +635,7 @@ impl Array { iter: T, ) -> VortexResult { let data = StructData::try_from_iter(iter)?; - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) } @@ -644,7 +643,7 @@ impl Array { /// Create a fieldless `StructArray` with the given length. pub fn new_fieldless_with_len(len: usize) -> Self { let data = StructData::new_fieldless_with_len(len); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) .vortex_expect("StructData is always valid") @@ -654,7 +653,7 @@ impl Array { impl StructData { pub fn with_column(&self, name: impl Into, array: ArrayRef) -> VortexResult { let name = name.into(); - let struct_dtype = self.struct_fields().clone(); + let struct_dtype = self.struct_fields(); let names = struct_dtype.names().iter().cloned().chain(once(name)); let types = struct_dtype.fields().chain(once(array.dtype().clone())); diff --git a/vortex-array/src/arrays/struct_/compute/mask.rs b/vortex-array/src/arrays/struct_/compute/mask.rs index cdf8a772bbf..c17493c11bf 100644 --- a/vortex-array/src/arrays/struct_/compute/mask.rs +++ b/vortex-array/src/arrays/struct_/compute/mask.rs @@ -15,7 +15,7 @@ impl MaskReduce for Struct { fn mask(array: ArrayView<'_, Struct>, mask: &ArrayRef) -> VortexResult> { StructArray::try_new_with_dtype( array.unmasked_fields().iter().cloned().collect::>(), - array.struct_fields().clone(), + array.struct_fields(), array.len(), array.validity().and(Validity::Array(mask.clone()))?, ) diff --git a/vortex-array/src/arrays/struct_/compute/slice.rs b/vortex-array/src/arrays/struct_/compute/slice.rs index 8d4a4a69737..0f36d87c560 100644 --- a/vortex-array/src/arrays/struct_/compute/slice.rs +++ b/vortex-array/src/arrays/struct_/compute/slice.rs @@ -25,7 +25,7 @@ impl SliceReduce for Struct { unsafe { StructArray::new_unchecked( fields, - array.struct_fields().clone(), + array.struct_fields(), range.len(), array.validity().slice(range)?, ) diff --git a/vortex-array/src/arrays/struct_/compute/take.rs b/vortex-array/src/arrays/struct_/compute/take.rs index 8836b45a2aa..8ce8b3f3851 100644 --- a/vortex-array/src/arrays/struct_/compute/take.rs +++ b/vortex-array/src/arrays/struct_/compute/take.rs @@ -20,7 +20,7 @@ impl TakeReduce for Struct { if array.is_empty() { return StructArray::try_new_with_dtype( array.iter_unmasked_fields().cloned().collect::>(), - array.struct_fields().clone(), + array.struct_fields(), indices.len(), Validity::AllInvalid, ) @@ -41,7 +41,7 @@ impl TakeReduce for Struct { .iter_unmasked_fields() .map(|field| field.take(inner_indices.clone())) .collect::, _>>()?, - array.struct_fields().clone(), + array.struct_fields(), indices.len(), array.validity().take(indices)?, ) diff --git a/vortex-array/src/arrays/struct_/mod.rs b/vortex-array/src/arrays/struct_/mod.rs index 2a9d6e1df07..65b53e2d603 100644 --- a/vortex-array/src/arrays/struct_/mod.rs +++ b/vortex-array/src/arrays/struct_/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::StructDataParts; pub use array::StructData; +pub use array::StructDataParts; pub use vtable::StructArray; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/struct_/vtable/mod.rs b/vortex-array/src/arrays/struct_/vtable/mod.rs index 56b263d91dc..3d32534cd3f 100644 --- a/vortex-array/src/arrays/struct_/vtable/mod.rs +++ b/vortex-array/src/arrays/struct_/vtable/mod.rs @@ -67,7 +67,7 @@ impl VTable for Struct { fn validate(&self, data: &StructData, dtype: &DType, len: usize) -> VortexResult<()> { match dtype { - DType::Struct(_, _) => {} + DType::Struct(..) => {} _ => vortex_bail!("Expected struct dtype, found {:?}", dtype), } if data.len() != len { @@ -111,7 +111,10 @@ impl VTable for Struct { _session: &VortexSession, ) -> VortexResult { if !metadata.is_empty() { - vortex_bail!("StructArray expects empty metadata, got {} bytes", metadata.len()); + vortex_bail!( + "StructArray expects empty metadata, got {} bytes", + metadata.len() + ); } let DType::Struct(struct_dtype, nullability) = dtype else { vortex_bail!("Expected struct dtype, found {:?}", dtype) diff --git a/vortex-array/src/arrays/varbin/array.rs b/vortex-array/src/arrays/varbin/array.rs index a4b041eca84..f91e1d5e9b4 100644 --- a/vortex-array/src/arrays/varbin/array.rs +++ b/vortex-array/src/arrays/varbin/array.rs @@ -402,7 +402,7 @@ impl VarBinData { impl Array { pub fn from_vec>(vec: Vec, dtype: DType) -> Self { let data = VarBinData::from_vec(vec, dtype); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") @@ -417,7 +417,7 @@ impl Array { dtype: DType, ) -> Self { let data = VarBinData::from_iter(iter, dtype); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") @@ -428,7 +428,7 @@ impl Array { dtype: DType, ) -> Self { let data = VarBinData::from_iter_nonnull(iter, dtype); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") @@ -495,14 +495,13 @@ impl VarBinData { .vortex_expect("VarBinArray offsets slot"); (self.dtype(), self.bytes, offsets, validity) } - } impl Array { /// Creates a new `VarBinArray`. pub fn new(offsets: ArrayRef, bytes: ByteBuffer, dtype: DType, validity: Validity) -> Self { let data = VarBinData::new(offsets, bytes, dtype, validity); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") @@ -520,7 +519,7 @@ impl Array { validity: Validity, ) -> Self { let data = unsafe { VarBinData::new_unchecked(offsets, bytes, dtype, validity) }; - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") @@ -539,7 +538,7 @@ impl Array { ) -> Self { let data = unsafe { VarBinData::new_unchecked_from_handle(offsets, bytes, dtype, validity) }; - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") @@ -553,7 +552,7 @@ impl Array { validity: Validity, ) -> VortexResult { let data = VarBinData::try_new(offsets, bytes, dtype, validity)?; - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) } @@ -636,7 +635,7 @@ impl<'a> FromIterator> for VarBinData { impl From> for Array { fn from(value: Vec<&[u8]>) -> Self { let data = VarBinData::from(value); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") @@ -646,7 +645,7 @@ impl From> for Array { impl From>> for Array { fn from(value: Vec>) -> Self { let data = VarBinData::from(value); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") @@ -656,7 +655,7 @@ impl From>> for Array { impl From> for Array { fn from(value: Vec) -> Self { let data = VarBinData::from(value); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") @@ -666,7 +665,7 @@ impl From> for Array { impl From> for Array { fn from(value: Vec<&str>) -> Self { let data = VarBinData::from(value); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") @@ -676,7 +675,7 @@ impl From> for Array { impl From>> for Array { fn from(value: Vec>) -> Self { let data = VarBinData::from(value); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") @@ -686,7 +685,7 @@ impl From>> for Array { impl From>>> for Array { fn from(value: Vec>>) -> Self { let data = VarBinData::from(value); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") @@ -696,7 +695,7 @@ impl From>>> for Array { impl From>> for Array { fn from(value: Vec>) -> Self { let data = VarBinData::from(value); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") @@ -706,7 +705,7 @@ impl From>> for Array { impl From>> for Array { fn from(value: Vec>) -> Self { let data = VarBinData::from(value); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect("VarBinData is always valid") @@ -716,7 +715,7 @@ impl From>> for Array { impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { let data = >::from_iter(iter); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect(" is always valid") @@ -726,7 +725,7 @@ impl<'a> FromIterator> for Array { impl FromIterator>> for Array { fn from_iter>>>(iter: T) -> Self { let data = >::from_iter(iter); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect(" is always valid") @@ -736,7 +735,7 @@ impl FromIterator>> for Array { impl FromIterator> for Array { fn from_iter>>(iter: T) -> Self { let data = >::from_iter(iter); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect(" is always valid") @@ -746,7 +745,7 @@ impl FromIterator> for Array { impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { let data = >::from_iter(iter); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) .vortex_expect(" is always valid") diff --git a/vortex-array/src/arrays/varbinview/array.rs b/vortex-array/src/arrays/varbinview/array.rs index 1fd82e5f9f1..4a0e9febbe7 100644 --- a/vortex-array/src/arrays/varbinview/array.rs +++ b/vortex-array/src/arrays/varbinview/array.rs @@ -283,10 +283,10 @@ impl VarBinViewData { Self::dtype_parts(&dtype).vortex_expect("VarBinViewArray dtype must be utf8 or binary"); Self { slots, - views, - buffers, is_utf8, nullability, + buffers, + views, } } @@ -580,7 +580,7 @@ impl Array { dtype: DType, ) -> Self { let data = VarBinViewData::from_iter(iter, dtype); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") @@ -588,7 +588,7 @@ impl Array { pub fn from_iter_str, I: IntoIterator>(iter: I) -> Self { let data = VarBinViewData::from_iter_str(iter); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") @@ -598,7 +598,7 @@ impl Array { iter: I, ) -> Self { let data = VarBinViewData::from_iter_nullable_str(iter); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") @@ -606,7 +606,7 @@ impl Array { pub fn from_iter_bin, I: IntoIterator>(iter: I) -> Self { let data = VarBinViewData::from_iter_bin(iter); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") @@ -616,7 +616,7 @@ impl Array { iter: I, ) -> Self { let data = VarBinViewData::from_iter_nullable_bin(iter); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") @@ -630,7 +630,7 @@ impl Array { validity: Validity, ) -> VortexResult { let data = VarBinViewData::try_new(views, buffers, dtype, validity)?; - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) } @@ -647,7 +647,7 @@ impl Array { validity: Validity, ) -> Self { let data = unsafe { VarBinViewData::new_unchecked(views, buffers, dtype, validity) }; - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") @@ -661,7 +661,7 @@ impl Array { validity: Validity, ) -> Self { let data = VarBinViewData::new_handle(views, buffers, dtype, validity); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") @@ -679,7 +679,7 @@ impl Array { validity: Validity, ) -> Self { let data = unsafe { VarBinViewData::new_handle_unchecked(views, buffers, dtype, validity) }; - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect("VarBinViewData is always valid") @@ -715,7 +715,7 @@ impl<'a> FromIterator> for VarBinViewData { impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { let data = >::from_iter(iter); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect(" is always valid") @@ -725,7 +725,7 @@ impl<'a> FromIterator> for Array { impl FromIterator>> for Array { fn from_iter>>>(iter: T) -> Self { let data = >::from_iter(iter); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect(" is always valid") @@ -735,7 +735,7 @@ impl FromIterator>> for Array { impl FromIterator> for Array { fn from_iter>>(iter: T) -> Self { let data = >::from_iter(iter); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect(" is always valid") @@ -745,7 +745,7 @@ impl FromIterator> for Array { impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { let data = >::from_iter(iter); - let dtype = data.dtype().clone(); + let dtype = data.dtype(); let len = data.len(); Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) .vortex_expect(" is always valid") diff --git a/vortex-array/src/arrays/varbinview/mod.rs b/vortex-array/src/arrays/varbinview/mod.rs index 1a56329cda3..d2d0041424d 100644 --- a/vortex-array/src/arrays/varbinview/mod.rs +++ b/vortex-array/src/arrays/varbinview/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::VarBinViewDataParts; pub use array::VarBinViewData; +pub use array::VarBinViewDataParts; pub use vtable::VarBinViewArray; mod accessor; diff --git a/vortex-array/src/arrays/variant/mod.rs b/vortex-array/src/arrays/variant/mod.rs index 9cf59ac5197..1d9a60e51f3 100644 --- a/vortex-array/src/arrays/variant/mod.rs +++ b/vortex-array/src/arrays/variant/mod.rs @@ -65,8 +65,13 @@ impl Array { let dtype = DType::Variant(child.dtype().nullability()); let len = child.len(); let stats = child.statistics().to_owned(); - Array::try_from_parts(ArrayParts::new(Variant, dtype, len, VariantData::new(child))) - .map(|array| array.with_stats_set(stats)) - .vortex_expect("VariantData is always valid") + Array::try_from_parts(ArrayParts::new( + Variant, + dtype, + len, + VariantData::new(child), + )) + .map(|array| array.with_stats_set(stats)) + .vortex_expect("VariantData is always valid") } } diff --git a/vortex-array/src/display/extractors/metadata.rs b/vortex-array/src/display/extractors/metadata.rs index f1776f3d383..a355bafe63c 100644 --- a/vortex-array/src/display/extractors/metadata.rs +++ b/vortex-array/src/display/extractors/metadata.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::fmt; + use crate::ArrayRef; use crate::display::extractor::IndentedFormatter; use crate::display::extractor::TreeContext; diff --git a/vortex-array/src/scalar_fn/fns/select.rs b/vortex-array/src/scalar_fn/fns/select.rs index c7478824d52..3c5a7cb8caf 100644 --- a/vortex-array/src/scalar_fn/fns/select.rs +++ b/vortex-array/src/scalar_fn/fns/select.rs @@ -163,7 +163,7 @@ impl ScalarFnVTable for Select { } }?; - let dtype = result.dtype().clone(); + let dtype = result.dtype(); let len = result.len(); Array::try_from_parts(ArrayParts::new(Struct, dtype, len, result))? .into_array() diff --git a/vortex-btrblocks/src/schemes/rle.rs b/vortex-btrblocks/src/schemes/rle.rs index fbc509d46cd..42fd4295bb7 100644 --- a/vortex-btrblocks/src/schemes/rle.rs +++ b/vortex-btrblocks/src/schemes/rle.rs @@ -21,6 +21,8 @@ use vortex_compressor::scheme::SchemeId; use vortex_compressor::stats::FloatStats; use vortex_compressor::stats::IntegerStats; use vortex_error::VortexResult; +#[cfg(feature = "unstable_encodings")] +use vortex_fastlanes::Delta; use vortex_fastlanes::RLE; use crate::ArrayAndStats; @@ -305,6 +307,5 @@ fn try_compress_delta( let compressed_deltas = compressor.compress_child(&deltas.into_array(), parent_ctx, parent_id, child_index)?; - vortex_fastlanes::DeltaData::try_new(compressed_bases, compressed_deltas, 0, child.len()) - .map(IntoArray::into_array) + Delta::try_new(compressed_bases, compressed_deltas, 0, child.len()).map(IntoArray::into_array) } diff --git a/vortex-cuda/benches/date_time_parts_cuda.rs b/vortex-cuda/benches/date_time_parts_cuda.rs index a91111e3f8a..53f303d0b2c 100644 --- a/vortex-cuda/benches/date_time_parts_cuda.rs +++ b/vortex-cuda/benches/date_time_parts_cuda.rs @@ -24,8 +24,8 @@ use vortex::array::validity::Validity; use vortex::buffer::Buffer; use vortex::dtype::DType; use vortex::dtype::Nullability; +use vortex::encodings::datetime_parts::DateTimeParts; use vortex::encodings::datetime_parts::DateTimePartsArray; -use vortex::encodings::datetime_parts::DateTimePartsData; use vortex::error::VortexExpect; use vortex::extension::datetime::TimeUnit; use vortex::extension::datetime::Timestamp; @@ -45,11 +45,8 @@ fn make_datetimeparts_array(len: usize, time_unit: TimeUnit) -> DateTimePartsArr let dtype = DType::Extension(Timestamp::new(time_unit, Nullability::NonNullable).erased()); - DateTimePartsArray::try_from_data( - DateTimePartsData::try_new(dtype, days_arr, seconds_arr, subseconds_arr) - .vortex_expect("Failed to create DateTimePartsArray"), - ) - .vortex_expect("DateTimePartsData is always valid") + DateTimeParts::try_new(dtype, days_arr, seconds_arr, subseconds_arr) + .vortex_expect("Failed to create DateTimePartsArray") } fn benchmark_datetimeparts(c: &mut Criterion) { diff --git a/vortex-cuda/benches/dynamic_dispatch_cuda.rs b/vortex-cuda/benches/dynamic_dispatch_cuda.rs index 21537b51031..6d4bb42d619 100644 --- a/vortex-cuda/benches/dynamic_dispatch_cuda.rs +++ b/vortex-cuda/benches/dynamic_dispatch_cuda.rs @@ -30,6 +30,7 @@ use vortex::encodings::alp::ALPFloat; use vortex::encodings::alp::Exponents; use vortex::encodings::alp::alp_encode; use vortex::encodings::fastlanes::BitPackedData; +use vortex::encodings::fastlanes::FoR; use vortex::encodings::fastlanes::FoRData; use vortex::encodings::runend::RunEnd; use vortex::error::VortexExpect; @@ -187,9 +188,9 @@ fn bench_for_bitpacked(c: &mut Criterion) { .collect(); let prim = PrimitiveArray::new(Buffer::from(residuals), NonNullable); let bp = BitPackedData::encode(&prim.into_array(), bit_width).vortex_expect("bitpack"); - let for_arr = - FoRData::try_new(bp.into_array(), Scalar::from(reference)).vortex_expect("for"); - let array = for_arr.into_array(); + let array = FoR::try_new(bp.into_array(), Scalar::from(reference)) + .vortex_expect("for") + .into_array(); group.bench_with_input( BenchmarkId::new("dynamic_dispatch_u32", len_str), @@ -320,8 +321,8 @@ fn bench_dict_bp_codes_bp_for_values(c: &mut Criterion) { let dict_prim = PrimitiveArray::new(Buffer::from(dict_residuals), NonNullable); let dict_bp = BitPackedData::encode(&dict_prim.into_array(), dict_bit_width) .vortex_expect("bitpack dict"); - let dict_for = FoRData::try_new(dict_bp.into_array(), Scalar::from(dict_reference)) - .vortex_expect("for dict"); + let dict_for = + FoR::try_new(dict_bp.into_array(), Scalar::from(dict_reference)).vortex_expect("for dict"); for (len, len_str) in BENCH_ARGS { group.throughput(Throughput::Bytes((len * size_of::()) as u64)); @@ -384,7 +385,7 @@ fn bench_alp_for_bitpacked(c: &mut Criterion) { BitPackedData::encode(for_arr.encoded(), bit_width).vortex_expect("bitpack encode"); let tree = ALP::new( - FoRData::try_new(bp.into_array(), for_arr.reference_scalar().clone()) + FoR::try_new(bp.into_array(), for_arr.reference_scalar().clone()) .vortex_expect("for_new") .into_array(), exponents, diff --git a/vortex-cuda/benches/for_cuda.rs b/vortex-cuda/benches/for_cuda.rs index eb87eb6d770..cc5fab5e22c 100644 --- a/vortex-cuda/benches/for_cuda.rs +++ b/vortex-cuda/benches/for_cuda.rs @@ -26,8 +26,8 @@ use vortex::buffer::Buffer; use vortex::dtype::NativePType; use vortex::dtype::PType; use vortex::encodings::fastlanes::BitPackedData; +use vortex::encodings::fastlanes::FoR; use vortex::encodings::fastlanes::FoRArray; -use vortex::encodings::fastlanes::FoRData; use vortex::error::VortexExpect; use vortex::scalar::Scalar; use vortex::session::VortexSession; @@ -57,17 +57,10 @@ where if bp && T::PTYPE != PType::U8 { let child = BitPackedData::encode(&primitive_array, 8).vortex_expect("failed to bitpack"); - FoRArray::try_from_data( - FoRData::try_new(child.into_array(), reference.into()) - .vortex_expect("failed to create FoR array"), - ) - .vortex_expect("FoRData is always valid") + FoR::try_new(child.into_array(), reference.into()) + .vortex_expect("failed to create FoR array") } else { - FoRArray::try_from_data( - FoRData::try_new(primitive_array, reference.into()) - .vortex_expect("failed to create FoR array"), - ) - .vortex_expect("FoRData is always valid") + FoR::try_new(primitive_array, reference.into()).vortex_expect("failed to create FoR array") } } diff --git a/vortex-cuda/src/kernel/arrays/dict.rs b/vortex-cuda/src/kernel/arrays/dict.rs index de06c5fa234..a8437e0b85b 100644 --- a/vortex-cuda/src/kernel/arrays/dict.rs +++ b/vortex-cuda/src/kernel/arrays/dict.rs @@ -144,11 +144,7 @@ async fn execute_dict_decimal( ctx: &mut CudaExecutionCtx, ) -> VortexResult { let dtype = dict.dtype().clone(); - let DictDataParts { - values, - codes, - .. - } = dict.into_data().into_parts(); + let DictDataParts { values, codes, .. } = dict.into_data().into_parts(); // Execute codes to get them as primitives on the device let codes_prim = codes.execute_cuda(ctx).await?.into_primitive(); @@ -235,11 +231,7 @@ async fn execute_dict_varbinview( ctx: &mut CudaExecutionCtx, ) -> VortexResult { let dtype = dict.dtype().clone(); - let DictDataParts { - values, - codes, - .. - } = dict.into_data().into_parts(); + let DictDataParts { values, codes, .. } = dict.into_data().into_parts(); let codes_prim = codes.execute_cuda(ctx).await?.into_primitive(); let codes_ptype = codes_prim.ptype(); diff --git a/vortex-cuda/src/kernel/encodings/bitpacked.rs b/vortex-cuda/src/kernel/encodings/bitpacked.rs index 7ed4f658610..3bdff6b439d 100644 --- a/vortex-cuda/src/kernel/encodings/bitpacked.rs +++ b/vortex-cuda/src/kernel/encodings/bitpacked.rs @@ -199,7 +199,7 @@ mod tests { // Last two items should be patched let bp_with_patches = BitPacked::encode(&array.into_array(), bw)?; - assert!(bp_with_patches.patches().is_some()); + assert!(bp_with_patches.patches(bp_with_patches.len()).is_some()); let cpu_result = bp_with_patches.to_canonical()?.into_array(); @@ -230,7 +230,7 @@ mod tests { // Last two items should be patched let bp_with_patches = BitPacked::encode(&array.into_array(), 9)?; - assert!(bp_with_patches.patches().is_some()); + assert!(bp_with_patches.patches(bp_with_patches.len()).is_some()); let cpu_result = bp_with_patches.to_canonical()?.into_array(); diff --git a/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs b/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs index 88f0ad65f96..b5aa921a9b6 100644 --- a/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs +++ b/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs @@ -11,9 +11,9 @@ use vortex::array::arrays::DecimalArray; use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::encodings::decimal_byte_parts::DecimalByteParts; use vortex::encodings::decimal_byte_parts::DecimalBytePartsDataParts; +use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::error::vortex_bail; -use vortex::error::VortexExpect; use crate::CudaExecutionCtx; use crate::executor::CudaArrayExt; diff --git a/vortex-cuda/src/kernel/encodings/zstd.rs b/vortex-cuda/src/kernel/encodings/zstd.rs index f7044697dde..e5b7da77fec 100644 --- a/vortex-cuda/src/kernel/encodings/zstd.rs +++ b/vortex-cuda/src/kernel/encodings/zstd.rs @@ -373,9 +373,7 @@ mod tests { let zstd_array = Zstd::from_var_bin_view(&strings, 3, 0)?; - let cpu_result = zstd_array - .decompress(cuda_ctx.execution_ctx())? - .to_canonical()?; + let cpu_result = Zstd::decompress(&zstd_array, cuda_ctx.execution_ctx())?.to_canonical()?; let gpu_result = ZstdExecutor .execute(zstd_array.into_array(), &mut cuda_ctx) .await?; @@ -410,9 +408,7 @@ mod tests { // 14 strings and 3 values per frame = ceil(14/3) = 5 frames. let zstd_array = Zstd::from_var_bin_view(&strings, 3, 3)?; - let cpu_result = zstd_array - .decompress(cuda_ctx.execution_ctx())? - .to_canonical()?; + let cpu_result = Zstd::decompress(&zstd_array, cuda_ctx.execution_ctx())?.to_canonical()?; let gpu_result = ZstdExecutor .execute(zstd_array.into_array(), &mut cuda_ctx) .await?; diff --git a/vortex-python/src/arrays/py/array.rs b/vortex-python/src/arrays/py/array.rs index 858f9280305..1653ed4b2aa 100644 --- a/vortex-python/src/arrays/py/array.rs +++ b/vortex-python/src/arrays/py/array.rs @@ -67,9 +67,11 @@ impl IntoArray for PythonArray { let dtype = self.dtype.clone(); let len = self.len; let stats = StatsSet::from(self.stats.clone()); - Array::try_from_parts(ArrayParts::new(vtable, dtype, len, self)) - .map(|array| array.with_stats_set(stats)) - .expect("PythonArray metadata extracted from PyPythonArray must be valid") - .into_array() + match Array::try_from_parts(ArrayParts::new(vtable, dtype, len, self)) { + Ok(array) => array.with_stats_set(stats).into_array(), + Err(err) => unreachable!( + "PythonArray metadata extracted from PyPythonArray must be valid: {err}" + ), + } } } diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index d2928a81a8f..8a4cfa53f59 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -27,6 +27,7 @@ use vortex::array::session::ArraySession; use vortex::dtype::PType; use vortex::encodings::alp::RDEncoder; use vortex::encodings::alp::alp_encode; +use vortex::encodings::fastlanes::Delta; use vortex::encodings::fastlanes::DeltaData; use vortex::encodings::fastlanes::FoR; use vortex::encodings::fastlanes::delta_compress; @@ -36,6 +37,7 @@ use vortex::encodings::pco::Pco; use vortex::encodings::runend::RunEnd; use vortex::encodings::sequence::sequence_encode; use vortex::encodings::zigzag::zigzag_encode; +use vortex::encodings::zstd::Zstd; use vortex::encodings::zstd::ZstdData; use vortex_sequence::Sequence; use vortex_session::VortexSession; @@ -162,10 +164,9 @@ fn bench_delta_compress_u32(bencher: Bencher) { fn bench_delta_decompress_u32(bencher: Bencher) { let (uint_array, ..) = setup_primitive_arrays(); let (bases, deltas) = delta_compress(&uint_array, &mut SESSION.create_execution_ctx()).unwrap(); - let compressed = - DeltaData::try_new(bases.into_array(), deltas.into_array(), 0, uint_array.len()) - .unwrap() - .into_array(); + let compressed = Delta::try_new(bases.into_array(), deltas.into_array(), 0, uint_array.len()) + .unwrap() + .into_array(); with_byte_counter(bencher, NUM_VALUES * 4) .with_inputs(|| &compressed) @@ -327,9 +328,13 @@ fn bench_zstd_compress_u32(bencher: Bencher) { #[divan::bench(name = "zstd_decompress_u32")] fn bench_zstd_decompress_u32(bencher: Bencher) { let (uint_array, ..) = setup_primitive_arrays(); - let compressed = ZstdData::from_array(uint_array.into_array(), 3, 8192) - .unwrap() - .into_array(); + let dtype = uint_array.dtype().clone(); + let compressed = Zstd::try_new( + dtype, + ZstdData::from_array(uint_array.into_array(), 3, 8192).unwrap(), + ) + .unwrap() + .into_array(); with_byte_counter(bencher, NUM_VALUES * 4) .with_inputs(|| &compressed) @@ -408,9 +413,13 @@ fn bench_zstd_compress_string(bencher: Bencher) { fn bench_zstd_decompress_string(bencher: Bencher) { let varbinview_arr = VarBinViewArray::from_iter_str(gen_varbin_words(NUM_VALUES as usize, 0.00005)); - let compressed = ZstdData::from_array(varbinview_arr.clone().into_array(), 3, 8192) - .unwrap() - .into_array(); + let dtype = varbinview_arr.dtype().clone(); + let compressed = Zstd::try_new( + dtype, + ZstdData::from_array(varbinview_arr.clone().into_array(), 3, 8192).unwrap(), + ) + .unwrap() + .into_array(); let nbytes = varbinview_arr.into_array().nbytes() as u64; with_byte_counter(bencher, nbytes) From af6da6dbd979054822c429abe2e45a575fb19077 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Fri, 3 Apr 2026 11:46:09 -0400 Subject: [PATCH 08/15] Normalize Execution Signed-off-by: Nicholas Gates --- encodings/alp/src/alp/array.rs | 23 ++- encodings/alp/src/alp_rd/array.rs | 9 +- encodings/bytebool/src/array.rs | 10 +- encodings/datetime-parts/src/array.rs | 12 +- .../src/decimal_byte_parts/mod.rs | 4 +- encodings/fastlanes/public-api.lock | 112 ++++++++++++++- .../bitpacking/array/bitpack_decompress.rs | 40 +++--- .../fastlanes/src/bitpacking/array/mod.rs | 135 ++++++++++++++++-- .../src/bitpacking/array/unpack_iter.rs | 27 ++-- .../src/bitpacking/compute/filter.rs | 13 +- .../src/bitpacking/compute/is_constant.rs | 5 +- .../fastlanes/src/bitpacking/compute/take.rs | 11 +- encodings/fastlanes/src/bitpacking/mod.rs | 1 + .../fastlanes/src/bitpacking/vtable/mod.rs | 105 +++++++------- encodings/fastlanes/src/delta/vtable/mod.rs | 2 +- .../fastlanes/src/for/array/for_decompress.rs | 9 +- encodings/fastlanes/src/for/vtable/mod.rs | 2 +- encodings/fastlanes/src/rle/vtable/mod.rs | 6 +- encodings/fsst/src/array.rs | 5 +- encodings/parquet-variant/src/array.rs | 4 +- encodings/pco/src/array.rs | 3 +- encodings/runend/src/array.rs | 9 +- encodings/sequence/src/array.rs | 17 ++- encodings/sparse/src/lib.rs | 7 +- encodings/zigzag/src/array.rs | 2 +- encodings/zstd/src/array.rs | 3 +- vortex-array/src/array/typed.rs | 3 +- vortex-array/src/arrays/bool/array.rs | 7 +- vortex-array/src/arrays/chunked/array.rs | 34 ++--- vortex-array/src/arrays/constant/array.rs | 5 +- vortex-array/src/arrays/decimal/array.rs | 20 +-- vortex-array/src/arrays/dict/array.rs | 11 +- vortex-array/src/arrays/extension/array.rs | 5 +- vortex-array/src/arrays/filter/array.rs | 5 +- .../src/arrays/fixed_size_list/array.rs | 12 +- vortex-array/src/arrays/list/array.rs | 12 +- vortex-array/src/arrays/listview/array.rs | 11 +- .../src/arrays/listview/conversion.rs | 3 +- vortex-array/src/arrays/masked/array.rs | 2 +- vortex-array/src/arrays/null/mod.rs | 6 +- vortex-array/src/arrays/patched/array.rs | 7 +- .../src/arrays/primitive/array/mod.rs | 20 +-- vortex-array/src/arrays/scalar_fn/array.rs | 2 +- .../src/arrays/scalar_fn/vtable/mod.rs | 5 +- vortex-array/src/arrays/shared/array.rs | 10 +- vortex-array/src/arrays/slice/array.rs | 5 +- vortex-array/src/arrays/struct_/array.rs | 19 ++- vortex-array/src/arrays/varbin/array.rs | 125 ++++------------ vortex-array/src/arrays/varbinview/array.rs | 90 ++++-------- vortex-array/src/arrays/variant/mod.rs | 17 +-- 50 files changed, 571 insertions(+), 441 deletions(-) diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index 38d20f7c5c9..93dfb8d3932 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -452,13 +452,14 @@ impl ALP { pub fn new(encoded: ArrayRef, exponents: Exponents, patches: Option) -> ALPArray { let dtype = ALPData::logical_dtype(&encoded).vortex_expect("ALP encoded dtype"); let len = encoded.len(); - Array::try_from_parts(ArrayParts::new( - ALP, - dtype, - len, - ALPData::new(encoded, exponents, patches), - )) - .vortex_expect("ALPData is always valid") + unsafe { + Array::from_parts_unchecked(ArrayParts::new( + ALP, + dtype, + len, + ALPData::new(encoded, exponents, patches), + )) + } } pub fn try_new( @@ -469,7 +470,7 @@ impl ALP { let dtype = ALPData::logical_dtype(&encoded)?; let len = encoded.len(); let data = ALPData::try_new(encoded, exponents, patches)?; - Array::try_from_parts(ArrayParts::new(ALP, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(ALP, dtype, len, data)) }) } /// # Safety @@ -481,10 +482,8 @@ impl ALP { ) -> ALPArray { let dtype = ALPData::logical_dtype(&encoded).vortex_expect("ALP encoded dtype"); let len = encoded.len(); - Array::try_from_parts(ArrayParts::new(ALP, dtype, len, unsafe { - ALPData::new_unchecked(encoded, exponents, patches) - })) - .vortex_expect("ALPData is always valid") + let data = unsafe { ALPData::new_unchecked(encoded, exponents, patches) }; + unsafe { Array::from_parts_unchecked(ArrayParts::new(ALP, dtype, len, data)) } } } diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index 811b082ef5c..b16470372bc 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -379,7 +379,11 @@ impl ALPRD { right_bit_width, left_parts_patches, )?; - Array::try_from_parts(ArrayParts::new(ALPRD, logical_dtype, len, data)) + Ok( + unsafe { + Array::from_parts_unchecked(ArrayParts::new(ALPRD, logical_dtype, len, data)) + }, + ) } /// # Safety @@ -404,8 +408,7 @@ impl ALPRD { left_parts_patches, ) }; - Array::try_from_parts(ArrayParts::new(ALPRD, logical_dtype, len, data)) - .vortex_expect("ALPRDData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(ALPRD, logical_dtype, len, data)) } } } diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index 0449765b3a6..48448ba7b57 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -28,7 +28,6 @@ use vortex_array::vtable::ValidityVTableFromValidityHelper; use vortex_array::vtable::validity_to_child; use vortex_buffer::BitBuffer; use vortex_buffer::ByteBuffer; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -189,8 +188,7 @@ impl ByteBool { let dtype = DType::Bool(validity.nullability()); let data = ByteBoolData::new(buffer, validity); let len = data.len(); - Array::try_from_parts(ArrayParts::new(ByteBool, dtype, len, data)) - .vortex_expect("ByteBoolData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(ByteBool, dtype, len, data)) } } /// Construct a [`ByteBoolArray`] from a `Vec` and validity. @@ -198,8 +196,7 @@ impl ByteBool { let data = ByteBoolData::from_vec(data, validity); let dtype = DType::Bool(data.validity.nullability()); let len = data.len(); - Array::try_from_parts(ArrayParts::new(ByteBool, dtype, len, data)) - .vortex_expect("ByteBoolData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(ByteBool, dtype, len, data)) } } /// Construct a [`ByteBoolArray`] from optional bools. @@ -207,8 +204,7 @@ impl ByteBool { let data = ByteBoolData::from(data); let dtype = DType::Bool(data.validity.nullability()); let len = data.len(); - Array::try_from_parts(ArrayParts::new(ByteBool, dtype, len, data)) - .vortex_expect("ByteBoolData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(ByteBool, dtype, len, data)) } } } diff --git a/encodings/datetime-parts/src/array.rs b/encodings/datetime-parts/src/array.rs index 9783b665953..f59cec3dd11 100644 --- a/encodings/datetime-parts/src/array.rs +++ b/encodings/datetime-parts/src/array.rs @@ -234,7 +234,11 @@ impl DateTimeParts { ) -> VortexResult { let data = DateTimePartsData::try_new(dtype.clone(), days, seconds, subseconds)?; let len = data.len(); - Array::try_from_parts(ArrayParts::new(DateTimeParts, dtype, len, data)) + Ok( + unsafe { + Array::from_parts_unchecked(ArrayParts::new(DateTimeParts, dtype, len, data)) + }, + ) } /// Construct a [`DateTimePartsArray`] from a [`TemporalArray`]. @@ -242,7 +246,11 @@ impl DateTimeParts { let dtype = temporal.dtype().clone(); let data = DateTimePartsData::try_from(temporal)?; let len = data.len(); - Array::try_from_parts(ArrayParts::new(DateTimeParts, dtype, len, data)) + Ok( + unsafe { + Array::from_parts_unchecked(ArrayParts::new(DateTimeParts, dtype, len, data)) + }, + ) } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs index 67d243df1b9..8773b2e8184 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs @@ -266,7 +266,9 @@ impl DecimalByteParts { let data = DecimalBytePartsData::try_new(msp, decimal_dtype)?; let dtype = DType::Decimal(decimal_dtype, data.msp().dtype().nullability()); let len = data.len(); - Array::try_from_parts(ArrayParts::new(DecimalByteParts, dtype, len, data)) + Ok(unsafe { + Array::from_parts_unchecked(ArrayParts::new(DecimalByteParts, dtype, len, data)) + }) } } diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index ced1130a8f1..4f426089f50 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -78,15 +78,15 @@ pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::decode_into(&mut sel pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::initial(&mut self) -> core::option::Option<&mut [T]> -pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::new_with_strategy(strategy: S, packed: vortex_buffer::ByteBuffer, bit_width: usize, offset: usize, len: usize) -> Self - pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::trailer(&mut self) -> core::option::Option<&mut [T]> +pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::try_new_with_strategy(strategy: S, packed: vortex_buffer::ByteBuffer, bit_width: usize, offset: usize, len: usize) -> vortex_error::VortexResult + impl vortex_fastlanes::unpack_iter::UnpackedChunks pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::full_chunks(&mut self) -> vortex_fastlanes::unpack_iter::BitUnpackIterator<'_, T> -pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::new(array: &vortex_fastlanes::BitPackedData, len: usize) -> Self +pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::try_new(array: &vortex_fastlanes::BitPackedData, len: usize) -> vortex_error::VortexResult pub trait vortex_fastlanes::unpack_iter::BitPacked: vortex_array::dtype::ptype::PhysicalPType @@ -218,6 +218,12 @@ pub fn vortex_fastlanes::BitPackedData::packed(&self) -> &vortex_array::buffer:: pub fn vortex_fastlanes::BitPackedData::packed_slice(&self) -> &[T] +pub fn vortex_fastlanes::BitPackedData::patch_chunk_offsets(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_fastlanes::BitPackedData::patch_indices(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_fastlanes::BitPackedData::patch_values(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + pub fn vortex_fastlanes::BitPackedData::patches(&self, len: usize) -> core::option::Option pub fn vortex_fastlanes::BitPackedData::ptype(&self, dtype: &vortex_array::dtype::DType) -> vortex_array::dtype::ptype::PType @@ -226,10 +232,12 @@ pub fn vortex_fastlanes::BitPackedData::replace_patches(&mut self, patches: core pub fn vortex_fastlanes::BitPackedData::try_new(packed: vortex_array::buffer::BufferHandle, ptype: vortex_array::dtype::ptype::PType, validity: vortex_array::validity::Validity, patches: core::option::Option, bit_width: u8, length: usize, offset: u16) -> vortex_error::VortexResult -pub fn vortex_fastlanes::BitPackedData::unpacked_chunks(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_fastlanes::unpack_iter::BitUnpackedChunks +pub fn vortex_fastlanes::BitPackedData::unpacked_chunks(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult> pub fn vortex_fastlanes::BitPackedData::validity(&self, nullability: vortex_array::dtype::nullability::Nullability) -> vortex_array::validity::Validity +pub fn vortex_fastlanes::BitPackedData::validity_child(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + pub fn vortex_fastlanes::BitPackedData::validity_mask(&self, len: usize, nullability: vortex_array::dtype::nullability::Nullability) -> vortex_mask::Mask impl core::clone::Clone for vortex_fastlanes::BitPackedData @@ -558,6 +566,102 @@ impl core::fmt::Debug for vortex_fastlanes::RLEData pub fn vortex_fastlanes::RLEData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub trait vortex_fastlanes::BitPackedArrayExt + +pub fn vortex_fastlanes::BitPackedArrayExt::bit_width(&self) -> u8 + +pub fn vortex_fastlanes::BitPackedArrayExt::bitpacked_data(&self) -> &vortex_fastlanes::BitPackedData + +pub fn vortex_fastlanes::BitPackedArrayExt::bitpacked_dtype(&self) -> &vortex_array::dtype::DType + +pub fn vortex_fastlanes::BitPackedArrayExt::bitpacked_len(&self) -> usize + +pub fn vortex_fastlanes::BitPackedArrayExt::offset(&self) -> u16 + +pub fn vortex_fastlanes::BitPackedArrayExt::packed(&self) -> &vortex_array::buffer::BufferHandle + +pub fn vortex_fastlanes::BitPackedArrayExt::packed_slice(&self) -> &[T] + +pub fn vortex_fastlanes::BitPackedArrayExt::patch_chunk_offsets(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_fastlanes::BitPackedArrayExt::patch_indices(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_fastlanes::BitPackedArrayExt::patch_values(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_fastlanes::BitPackedArrayExt::patches(&self) -> core::option::Option + +pub fn vortex_fastlanes::BitPackedArrayExt::unpacked_chunks(&self) -> vortex_error::VortexResult> + +pub fn vortex_fastlanes::BitPackedArrayExt::validity(&self) -> vortex_array::validity::Validity + +pub fn vortex_fastlanes::BitPackedArrayExt::validity_child(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_fastlanes::BitPackedArrayExt::validity_mask(&self) -> vortex_mask::Mask + +impl vortex_fastlanes::BitPackedArrayExt for vortex_array::array::typed::Array + +pub fn vortex_array::array::typed::Array::bit_width(&self) -> u8 + +pub fn vortex_array::array::typed::Array::bitpacked_data(&self) -> &vortex_fastlanes::BitPackedData + +pub fn vortex_array::array::typed::Array::bitpacked_dtype(&self) -> &vortex_array::dtype::DType + +pub fn vortex_array::array::typed::Array::bitpacked_len(&self) -> usize + +pub fn vortex_array::array::typed::Array::offset(&self) -> u16 + +pub fn vortex_array::array::typed::Array::packed(&self) -> &vortex_array::buffer::BufferHandle + +pub fn vortex_array::array::typed::Array::packed_slice(&self) -> &[T] + +pub fn vortex_array::array::typed::Array::patch_chunk_offsets(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_array::array::typed::Array::patch_indices(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_array::array::typed::Array::patch_values(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_array::array::typed::Array::patches(&self) -> core::option::Option + +pub fn vortex_array::array::typed::Array::unpacked_chunks(&self) -> vortex_error::VortexResult> + +pub fn vortex_array::array::typed::Array::validity(&self) -> vortex_array::validity::Validity + +pub fn vortex_array::array::typed::Array::validity_child(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_array::array::typed::Array::validity_mask(&self) -> vortex_mask::Mask + +impl vortex_fastlanes::BitPackedArrayExt for vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked> + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::bit_width(&self) -> u8 + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::bitpacked_data(&self) -> &vortex_fastlanes::BitPackedData + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::bitpacked_dtype(&self) -> &vortex_array::dtype::DType + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::bitpacked_len(&self) -> usize + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::offset(&self) -> u16 + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::packed(&self) -> &vortex_array::buffer::BufferHandle + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::packed_slice(&self) -> &[T] + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::patch_chunk_offsets(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::patch_indices(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::patch_values(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::patches(&self) -> core::option::Option + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::unpacked_chunks(&self) -> vortex_error::VortexResult> + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::validity(&self) -> vortex_array::validity::Validity + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::validity_child(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::validity_mask(&self) -> vortex_mask::Mask + pub fn vortex_fastlanes::delta_compress(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<(vortex_array::arrays::primitive::vtable::PrimitiveArray, vortex_array::arrays::primitive::vtable::PrimitiveArray)> pub fn vortex_fastlanes::initialize(session: &mut vortex_session::VortexSession) diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs index 2f985b497a3..5ed8d6b89fa 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs @@ -19,6 +19,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::BitPacked; +use crate::BitPackedArrayExt; use crate::unpack_iter::BitPacked as BitPackedUnpack; /// Unpacks a bit-packed array into a primitive array. @@ -36,13 +37,13 @@ pub fn unpack_primitive_array( ctx: &mut ExecutionCtx, ) -> VortexResult { let mut builder = PrimitiveBuilder::with_capacity(array.dtype().nullability(), array.len()); - unpack_into_primitive_builder::(&array, &mut builder, ctx)?; + unpack_into_primitive_builder::(array, &mut builder, ctx)?; assert_eq!(builder.len(), array.len()); Ok(builder.finish_into_primitive()) } pub(crate) fn unpack_into_primitive_builder( - array: &ArrayView<'_, BitPacked>, + array: ArrayView<'_, BitPacked>, // TODO(ngates): do we want to use fastlanes alignment for this buffer? builder: &mut PrimitiveBuilder, ctx: &mut ExecutionCtx, @@ -57,16 +58,16 @@ pub(crate) fn unpack_into_primitive_builder( // SAFETY: We later initialize the the uninitialized range of values with `copy_from_slice`. unsafe { // Append a dense null Mask. - uninit_range.append_mask(array.validity_mask(array.len(), array.dtype().nullability())); + uninit_range.append_mask(array.validity_mask()); } // SAFETY: `decode_into` will initialize all values in this range. let uninit_slice = unsafe { uninit_range.slice_uninit_mut(0, array.len()) }; - let mut bit_packed_iter = array.unpacked_chunks(array.dtype(), array.len()); + let mut bit_packed_iter = array.unpacked_chunks()?; bit_packed_iter.decode_into(uninit_slice); - if let Some(ref patches) = array.patches(array.len()) { + if let Some(ref patches) = array.patches() { apply_patches_to_uninit_range(&mut uninit_range, patches, ctx)?; }; @@ -249,7 +250,7 @@ mod tests { .into_array() .to_primitive(); let bitpacked = encode(&zeros, 10); - assert!(bitpacked.patches(bitpacked.len()).is_some()); + assert!(bitpacked.patches().is_some()); let actual = unpack(&bitpacked)?; assert_arrays_eq!( actual, @@ -262,7 +263,7 @@ mod tests { fn test_one_full_chunk_and_one_short_chunk_no_patch() -> VortexResult<()> { let zeros = BufferMut::from_iter(0u16..1025).into_array().to_primitive(); let bitpacked = encode(&zeros, 11); - assert!(bitpacked.patches(bitpacked.len()).is_none()); + assert!(bitpacked.patches().is_none()); let actual = unpack(&bitpacked)?; assert_arrays_eq!(actual, PrimitiveArray::from_iter(0u16..1025)); Ok(()) @@ -275,7 +276,7 @@ mod tests { .to_primitive(); let bitpacked = encode(&zeros, 10); assert_eq!(bitpacked.len(), 1025); - assert!(bitpacked.patches(bitpacked.len()).is_some()); + assert!(bitpacked.patches().is_some()); let actual = unpack(&bitpacked)?; assert_arrays_eq!(actual, PrimitiveArray::from_iter(512u16..1537)); Ok(()) @@ -288,7 +289,7 @@ mod tests { .to_primitive(); let bitpacked = encode(&zeros, 10); assert_eq!(bitpacked.len(), 1025); - assert!(bitpacked.patches(bitpacked.len()).is_some()); + assert!(bitpacked.patches().is_some()); let slice_ref = bitpacked.into_array().slice(1023..1025).unwrap(); let actual = { let mut ctx = SESSION.create_execution_ctx(); @@ -308,7 +309,7 @@ mod tests { .to_primitive(); let bitpacked = encode(&zeros, 10); assert_eq!(bitpacked.len(), 2229); - assert!(bitpacked.patches(bitpacked.len()).is_some()); + assert!(bitpacked.patches().is_some()); let slice_ref = bitpacked.into_array().slice(1023..2049).unwrap(); let actual = { let mut ctx = SESSION.create_execution_ctx(); @@ -331,7 +332,7 @@ mod tests { let mut builder = PrimitiveBuilder::::new(Nullability::NonNullable); unpack_into_primitive_builder( - &bitpacked.as_view(), + bitpacked.as_view(), &mut builder, &mut SESSION.create_execution_ctx(), )?; @@ -359,7 +360,7 @@ mod tests { // Unpack into a new builder. let mut builder = PrimitiveBuilder::::with_capacity(Nullability::Nullable, 5); unpack_into_primitive_builder( - &bitpacked.as_view(), + bitpacked.as_view(), &mut builder, &mut SESSION.create_execution_ctx(), )?; @@ -388,14 +389,14 @@ mod tests { // Bitpack with a bit width that will require patches. let bitpacked = encode(&array, 4); assert!( - bitpacked.patches(bitpacked.len()).is_some(), + bitpacked.patches().is_some(), "Should have patches for values > 15" ); // Unpack into a new builder. let mut builder = PrimitiveBuilder::::with_capacity(Nullability::NonNullable, 100); unpack_into_primitive_builder( - &bitpacked.as_view(), + bitpacked.as_view(), &mut builder, &mut SESSION.create_execution_ctx(), )?; @@ -420,10 +421,7 @@ mod tests { // Bitpack with a small bit width to force patches. let bitpacked = encode(&array, 6); - assert!( - bitpacked.patches(bitpacked.len()).is_some(), - "Should have patches" - ); + assert!(bitpacked.patches().is_some(), "Should have patches"); // Test with a larger array with multiple patches across chunks. let large_values: Vec = (0..3072) @@ -437,7 +435,7 @@ mod tests { .collect(); let large_array = PrimitiveArray::from_iter(large_values); let large_bitpacked = encode(&large_array, 8); - assert!(large_bitpacked.patches(large_bitpacked.len()).is_some()); + assert!(large_bitpacked.patches().is_some()); let large_result = unpack(&large_bitpacked)?; assert_eq!(large_result.len(), 3072); @@ -468,7 +466,7 @@ mod tests { let patch_array = PrimitiveArray::new(patch_values, patch_validity); let patch_bitpacked = encode(&patch_array, 5); - assert!(patch_bitpacked.patches(patch_bitpacked.len()).is_some()); + assert!(patch_bitpacked.patches().is_some()); let patch_result = unpack(&patch_bitpacked).vortex_expect("unpack"); assert_eq!(patch_result.len(), 7); @@ -625,7 +623,7 @@ mod tests { .collect(); let boundary_array = PrimitiveArray::from_iter(boundary_values); let boundary_bp = encode(&boundary_array, 7); - assert!(boundary_bp.patches(boundary_bp.len()).is_some()); + assert!(boundary_bp.patches().is_some()); let boundary_result = unpack(&boundary_bp)?; assert_eq!(boundary_result.len(), 3072); diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index 0977bb8b34e..9e8c98b1c04 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -2,7 +2,9 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use fastlanes::BitPacking; +use vortex_array::Array; use vortex_array::ArrayRef; +use vortex_array::ArrayView; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; @@ -281,14 +283,38 @@ impl BitPackedData { unsafe { std::slice::from_raw_parts(packed_ptr, packed_len) } } + #[inline] + pub fn patch_indices(&self) -> Option<&ArrayRef> { + self.slots[PATCH_INDICES_SLOT].as_ref() + } + + #[inline] + pub fn patch_values(&self) -> Option<&ArrayRef> { + self.slots[PATCH_VALUES_SLOT].as_ref() + } + + #[inline] + pub fn patch_chunk_offsets(&self) -> Option<&ArrayRef> { + self.slots[PATCH_CHUNK_OFFSETS_SLOT].as_ref() + } + + #[inline] + pub fn validity_child(&self) -> Option<&ArrayRef> { + self.slots[VALIDITY_SLOT].as_ref() + } + /// Accessor for bit unpacked chunks - pub fn unpacked_chunks(&self, dtype: &DType, len: usize) -> BitUnpackedChunks { + pub fn unpacked_chunks( + &self, + dtype: &DType, + len: usize, + ) -> VortexResult> { assert_eq!( T::PTYPE, self.ptype(dtype), "Requested type doesn't match the array ptype" ); - BitUnpackedChunks::new(self, len) + BitUnpackedChunks::try_new(self, len) } /// Bit-width of the packed values @@ -303,10 +329,7 @@ impl BitPackedData { /// If present, patches MUST be a `SparseArray` with equal-length to this array, and whose /// indices indicate the locations of patches. The indices must have non-zero length. pub fn patches(&self, len: usize) -> Option { - match ( - &self.slots[PATCH_INDICES_SLOT], - &self.slots[PATCH_VALUES_SLOT], - ) { + match (self.patch_indices(), self.patch_values()) { (Some(indices), Some(values)) => { let patch_offset = self .patch_offset @@ -317,7 +340,7 @@ impl BitPackedData { patch_offset, indices.clone(), values.clone(), - self.slots[PATCH_CHUNK_OFFSETS_SLOT].clone(), + self.patch_chunk_offsets().cloned(), self.patch_offset_within_chunk, ) }) @@ -328,7 +351,7 @@ impl BitPackedData { /// Returns the validity, reconstructed from the stored slot. pub fn validity(&self, nullability: Nullability) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], nullability) + child_to_validity(&self.validity_child().cloned(), nullability) } pub fn replace_patches(&mut self, patches: Option) { @@ -393,6 +416,102 @@ impl BitPackedData { } } +pub trait BitPackedArrayExt { + fn bitpacked_data(&self) -> &BitPackedData; + fn bitpacked_dtype(&self) -> &DType; + fn bitpacked_len(&self) -> usize; + + #[inline] + fn packed(&self) -> &BufferHandle { + self.bitpacked_data().packed() + } + + #[inline] + fn bit_width(&self) -> u8 { + self.bitpacked_data().bit_width() + } + + #[inline] + fn offset(&self) -> u16 { + self.bitpacked_data().offset() + } + + #[inline] + fn patch_indices(&self) -> Option<&ArrayRef> { + self.bitpacked_data().patch_indices() + } + + #[inline] + fn patch_values(&self) -> Option<&ArrayRef> { + self.bitpacked_data().patch_values() + } + + #[inline] + fn patch_chunk_offsets(&self) -> Option<&ArrayRef> { + self.bitpacked_data().patch_chunk_offsets() + } + + #[inline] + fn validity_child(&self) -> Option<&ArrayRef> { + self.bitpacked_data().validity_child() + } + + #[inline] + fn patches(&self) -> Option { + self.bitpacked_data().patches(self.bitpacked_len()) + } + + #[inline] + fn validity(&self) -> Validity { + self.bitpacked_data() + .validity(self.bitpacked_dtype().nullability()) + } + + #[inline] + fn validity_mask(&self) -> vortex_mask::Mask { + self.validity().to_mask(self.bitpacked_len()) + } + + #[inline] + fn packed_slice(&self) -> &[T] { + self.bitpacked_data().packed_slice::() + } + + #[inline] + fn unpacked_chunks(&self) -> VortexResult> { + self.bitpacked_data() + .unpacked_chunks::(self.bitpacked_dtype(), self.bitpacked_len()) + } +} + +impl BitPackedArrayExt for Array { + fn bitpacked_data(&self) -> &BitPackedData { + self.data() + } + + fn bitpacked_dtype(&self) -> &DType { + self.dtype() + } + + fn bitpacked_len(&self) -> usize { + self.len() + } +} + +impl BitPackedArrayExt for ArrayView<'_, crate::BitPacked> { + fn bitpacked_data(&self) -> &BitPackedData { + self.data() + } + + fn bitpacked_dtype(&self) -> &DType { + self.dtype() + } + + fn bitpacked_len(&self) -> usize { + self.len() + } +} + #[cfg(test)] mod test { use vortex_array::IntoArray; diff --git a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs index 8781a107e4d..80a73f82678 100644 --- a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs +++ b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs @@ -11,6 +11,8 @@ use lending_iterator::prelude::Item; use lending_iterator::prelude::LendingIterator; use vortex_array::dtype::PhysicalPType; use vortex_buffer::ByteBuffer; +use vortex_error::VortexResult; +use vortex_error::vortex_ensure; use crate::BitPackedData; @@ -55,11 +57,11 @@ impl> UnpackStrategy for BitPackingStr /// use vortex_array::IntoArray; /// use vortex_buffer::buffer; /// use vortex_fastlanes::BitPackedData; +/// use vortex_fastlanes::BitPackedArrayExt; /// use vortex_fastlanes::unpack_iter::BitUnpackedChunks; /// /// let array = BitPackedData::encode(&buffer![2, 3, 4, 5].into_array(), 2).unwrap(); -/// let mut unpacked_chunks: BitUnpackedChunks = -/// array.unpacked_chunks(array.dtype(), array.len()); +/// let mut unpacked_chunks: BitUnpackedChunks = array.unpacked_chunks().unwrap(); /// /// if let Some(header) = unpacked_chunks.initial() { /// // handle partial initial chunk @@ -90,8 +92,8 @@ pub struct UnpackedChunks> { pub type BitUnpackedChunks = UnpackedChunks; impl BitUnpackedChunks { - pub fn new(array: &BitPackedData, len: usize) -> Self { - Self::new_with_strategy( + pub fn try_new(array: &BitPackedData, len: usize) -> VortexResult { + Self::try_new_with_strategy( BitPackingStrategy, array.packed().clone().unwrap_host(), array.bit_width() as usize, @@ -116,26 +118,29 @@ impl BitUnpackedChunks { } impl> UnpackedChunks { - pub fn new_with_strategy( + pub fn try_new_with_strategy( strategy: S, packed: ByteBuffer, bit_width: usize, offset: usize, len: usize, - ) -> Self { + ) -> VortexResult { + vortex_ensure!( + offset < CHUNK_SIZE, + "Invalid bit-packed offset {offset}, expected < {CHUNK_SIZE}" + ); let elems_per_chunk = 128 * bit_width / size_of::(); let num_chunks = (offset + len).div_ceil(CHUNK_SIZE); - assert_eq!( - packed.len() / size_of::(), - num_chunks * elems_per_chunk, + vortex_ensure!( + packed.len() / size_of::() == num_chunks * elems_per_chunk, "Invalid packed length: got {}, expected {}", packed.len() / size_of::(), num_chunks * elems_per_chunk ); let last_chunk_length = (offset + len) % CHUNK_SIZE; - Self { + Ok(Self { strategy, bit_width, offset, @@ -144,7 +149,7 @@ impl> UnpackedChunks { buffer: [const { MaybeUninit::::uninit() }; CHUNK_SIZE], num_chunks, last_chunk_length, - } + }) } #[inline(always)] diff --git a/encodings/fastlanes/src/bitpacking/compute/filter.rs b/encodings/fastlanes/src/bitpacking/compute/filter.rs index 52987be657c..5ebeaa4705c 100644 --- a/encodings/fastlanes/src/bitpacking/compute/filter.rs +++ b/encodings/fastlanes/src/bitpacking/compute/filter.rs @@ -25,6 +25,7 @@ use vortex_mask::MaskValues; use super::chunked_indices; use super::take::UNPACK_CHUNK_THRESHOLD; use crate::BitPacked; +use crate::BitPackedArrayExt; use crate::BitPackedData; /// The threshold over which it is faster to fully unpack the entire [`BitPackedArray`] and then @@ -65,7 +66,7 @@ impl FilterKernel for BitPacked { // Filter and patch using the correct unsigned type for FastLanes, then cast to signed if needed. let primitive = match_each_unsigned_integer_ptype!(array.dtype().as_ptype().to_unsigned(), |U| { - let (buffer, validity) = filter_primitive_without_patches::(&array, values)?; + let (buffer, validity) = filter_primitive_without_patches::(array, values)?; // reinterpret_cast for signed types. let primitive = PrimitiveArray::new(buffer, validity); if array.dtype().as_ptype().is_signed_int() { @@ -80,7 +81,7 @@ impl FilterKernel for BitPacked { }); let patches = array - .patches(array.len()) + .patches() .map(|patches| patches.filter(&Mask::Values(values.clone()), ctx)) .transpose()? .flatten(); @@ -107,13 +108,11 @@ impl FilterKernel for BitPacked { /// /// Returns a tuple of (values buffer, validity mask). fn filter_primitive_without_patches( - array: &ArrayView<'_, BitPacked>, + array: ArrayView<'_, BitPacked>, selection: &Arc, ) -> VortexResult<(Buffer, Validity)> { - let values = filter_with_indices(array, selection.indices()); - let validity = array - .validity(array.dtype().nullability()) - .filter(&Mask::Values(selection.clone()))?; + let values = filter_with_indices(array.data(), selection.indices()); + let validity = array.validity().filter(&Mask::Values(selection.clone()))?; Ok((values.freeze(), validity)) } diff --git a/encodings/fastlanes/src/bitpacking/compute/is_constant.rs b/encodings/fastlanes/src/bitpacking/compute/is_constant.rs index 548f897432d..900284e926d 100644 --- a/encodings/fastlanes/src/bitpacking/compute/is_constant.rs +++ b/encodings/fastlanes/src/bitpacking/compute/is_constant.rs @@ -22,6 +22,7 @@ use vortex_array::scalar::Scalar; use vortex_error::VortexResult; use crate::BitPacked; +use crate::BitPackedArrayExt; use crate::unpack_iter::BitPacked as BitPackedUnpack; /// BitPacked-specific is_constant kernel with SIMD support. @@ -54,8 +55,8 @@ impl DynAggregateKernel for BitPackedIsConstantKernel { fn bitpacked_is_constant( array: ArrayView<'_, BitPacked>, ) -> VortexResult { - let mut bit_unpack_iterator = array.unpacked_chunks::(array.dtype(), array.len()); - let patches = array.patches(array.len()).map(|p| { + let mut bit_unpack_iterator = array.unpacked_chunks::()?; + let patches = array.patches().map(|p| { let values = p.values().to_primitive(); let indices = p.indices().to_primitive(); let offset = p.offset(); diff --git a/encodings/fastlanes/src/bitpacking/compute/take.rs b/encodings/fastlanes/src/bitpacking/compute/take.rs index a762d904323..6d652c42b42 100644 --- a/encodings/fastlanes/src/bitpacking/compute/take.rs +++ b/encodings/fastlanes/src/bitpacking/compute/take.rs @@ -24,6 +24,7 @@ use vortex_error::VortexResult; use super::chunked_indices; use crate::BitPacked; +use crate::BitPackedArrayExt; use crate::bitpack_decompress; // TODO(connor): This is duplicated in `encodings/fastlanes/src/bitpacking/kernels/mod.rs`. @@ -47,13 +48,13 @@ impl TakeExecute for BitPacked { // NOTE: we use the unsigned PType because all values in the BitPackedArray must // be non-negative (pre-condition of creating the BitPackedArray). let ptype: PType = PType::try_from(array.dtype())?; - let validity = array.validity(array.dtype().nullability()); + let validity = array.validity(); let taken_validity = validity.take(indices)?; let indices = indices.clone().execute::(ctx)?; let taken = match_each_unsigned_integer_ptype!(ptype.to_unsigned(), |T| { match_each_integer_ptype!(indices.ptype(), |I| { - take_primitive::(&array, &indices, taken_validity, ctx)? + take_primitive::(array, &indices, taken_validity, ctx)? }) }); let taken = if ptype.is_signed_int() { @@ -70,7 +71,7 @@ impl TakeExecute for BitPacked { } fn take_primitive( - array: &ArrayView<'_, BitPacked>, + array: ArrayView<'_, BitPacked>, indices: &PrimitiveArray, taken_validity: Validity, ctx: &mut ExecutionCtx, @@ -146,7 +147,7 @@ fn take_primitive( } else { PrimitiveArray::new(output, taken_validity) }; - if let Some(patches) = array.patches(array.len()) + if let Some(patches) = array.patches() && let Some(patches) = patches.take(&indices.clone().into_array(), ctx)? { let cast_patches = patches.cast_values(unpatched_taken.dtype())?; @@ -255,7 +256,7 @@ mod test { BitPackedData::encode(&buffer![1i32, 2i32, 3i32, 4i32].into_array(), 1).unwrap(); let taken_primitive = take_primitive::( - &start.as_view(), + start.as_view(), &PrimitiveArray::from_iter([0u64, 1, 2, 3]), Validity::NonNullable, &mut LEGACY_SESSION.create_execution_ctx(), diff --git a/encodings/fastlanes/src/bitpacking/mod.rs b/encodings/fastlanes/src/bitpacking/mod.rs index d777d372773..95d8ef8c608 100644 --- a/encodings/fastlanes/src/bitpacking/mod.rs +++ b/encodings/fastlanes/src/bitpacking/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::BitPackedArrayExt; pub use array::BitPackedData; pub use array::BitPackedDataParts; pub use array::bitpack_compress; diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs index 4ac2d748f48..317788ce9c5 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs @@ -37,6 +37,7 @@ use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use crate::BitPackedArrayExt; use crate::BitPackedData; use crate::BitPackedDataParts; use crate::bitpack_decompress::unpack_array; @@ -88,10 +89,34 @@ impl VTable for BitPacked { array.offset.hash(state); array.bit_width.hash(state); array.packed.array_hash(state, precision); - option_array_hash(&array.slots[PATCH_INDICES_SLOT], state, precision); - option_array_hash(&array.slots[PATCH_VALUES_SLOT], state, precision); - option_array_hash(&array.slots[PATCH_CHUNK_OFFSETS_SLOT], state, precision); - option_array_hash(&array.slots[VALIDITY_SLOT], state, precision); + match array.patch_indices() { + Some(indices) => { + true.hash(state); + indices.array_hash(state, precision); + } + None => false.hash(state), + } + match array.patch_values() { + Some(values) => { + true.hash(state); + values.array_hash(state, precision); + } + None => false.hash(state), + } + match array.patch_chunk_offsets() { + Some(offsets) => { + true.hash(state); + offsets.array_hash(state, precision); + } + None => false.hash(state), + } + match array.validity_child() { + Some(validity) => { + true.hash(state); + validity.array_hash(state, precision); + } + None => false.hash(state), + } array.patch_offset.hash(state); array.patch_offset_within_chunk.hash(state); } @@ -100,26 +125,26 @@ impl VTable for BitPacked { array.offset == other.offset && array.bit_width == other.bit_width && array.packed.array_eq(&other.packed, precision) - && option_array_eq( - &array.slots[PATCH_INDICES_SLOT], - &other.slots[PATCH_INDICES_SLOT], - precision, - ) - && option_array_eq( - &array.slots[PATCH_VALUES_SLOT], - &other.slots[PATCH_VALUES_SLOT], - precision, - ) - && option_array_eq( - &array.slots[PATCH_CHUNK_OFFSETS_SLOT], - &other.slots[PATCH_CHUNK_OFFSETS_SLOT], - precision, - ) - && option_array_eq( - &array.slots[VALIDITY_SLOT], - &other.slots[VALIDITY_SLOT], - precision, - ) + && match (array.patch_indices(), other.patch_indices()) { + (Some(lhs), Some(rhs)) => lhs.array_eq(rhs, precision), + (None, None) => true, + _ => false, + } + && match (array.patch_values(), other.patch_values()) { + (Some(lhs), Some(rhs)) => lhs.array_eq(rhs, precision), + (None, None) => true, + _ => false, + } + && match (array.patch_chunk_offsets(), other.patch_chunk_offsets()) { + (Some(lhs), Some(rhs)) => lhs.array_eq(rhs, precision), + (None, None) => true, + _ => false, + } + && match (array.validity_child(), other.validity_child()) { + (Some(lhs), Some(rhs)) => lhs.array_eq(rhs, precision), + (None, None) => true, + _ => false, + } && array.patch_offset == other.patch_offset && array.patch_offset_within_chunk == other.patch_offset_within_chunk } @@ -182,7 +207,7 @@ impl VTable for BitPacked { bit_width: array.bit_width() as u32, offset: array.offset() as u32, patches: array - .patches(array.len()) + .patches() .map(|p| p.to_metadata(array.len(), array.dtype())) .transpose()?, } @@ -271,7 +296,7 @@ impl VTable for BitPacked { ) -> VortexResult<()> { match_each_integer_ptype!(array.dtype().as_ptype(), |T| { unpack_into_primitive_builder::( - &array, + array, builder .as_any_mut() .downcast_mut() @@ -284,14 +309,14 @@ impl VTable for BitPacked { fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { require_patches!( array, - array.patches(array.len()), + array.patches(), PATCH_INDICES_SLOT, PATCH_VALUES_SLOT, PATCH_CHUNK_OFFSETS_SLOT ); require_validity!( array, - &array.validity(array.dtype().nullability()), + &array.validity(), VALIDITY_SLOT => AnyCanonical ); @@ -328,7 +353,7 @@ impl BitPacked { let dtype = DType::Primitive(ptype, validity.nullability()); let data = BitPackedData::try_new(packed, ptype, validity, patches, bit_width, len, offset)?; - Array::try_from_parts(ArrayParts::new(BitPacked, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(BitPacked, dtype, len, data)) }) } pub fn into_parts(array: BitPackedArray) -> BitPackedDataParts { @@ -342,25 +367,3 @@ impl BitPacked { BitPackedData::encode(array, bit_width) } } - -fn option_array_hash( - array: &Option, - state: &mut H, - precision: Precision, -) { - match array { - Some(array) => { - true.hash(state); - array.array_hash(state, precision); - } - None => false.hash(state), - } -} - -fn option_array_eq(lhs: &Option, rhs: &Option, precision: Precision) -> bool { - match (lhs, rhs) { - (Some(lhs), Some(rhs)) => lhs.array_eq(rhs, precision), - (None, None) => true, - _ => false, - } -} diff --git a/encodings/fastlanes/src/delta/vtable/mod.rs b/encodings/fastlanes/src/delta/vtable/mod.rs index ba31e1a13c8..746221a5b1e 100644 --- a/encodings/fastlanes/src/delta/vtable/mod.rs +++ b/encodings/fastlanes/src/delta/vtable/mod.rs @@ -183,7 +183,7 @@ impl Delta { ) -> VortexResult { let dtype = bases.dtype().with_nullability(deltas.dtype().nullability()); let data = DeltaData::try_new(bases, deltas, offset, len)?; - Array::try_from_parts(ArrayParts::new(Delta, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Delta, dtype, len, data)) }) } /// Compress a primitive array using Delta encoding. diff --git a/encodings/fastlanes/src/for/array/for_decompress.rs b/encodings/fastlanes/src/for/array/for_decompress.rs index 639d369a1fc..82a84b73fa4 100644 --- a/encodings/fastlanes/src/for/array/for_decompress.rs +++ b/encodings/fastlanes/src/for/array/for_decompress.rs @@ -18,6 +18,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::BitPacked; +use crate::BitPackedArrayExt; use crate::FoRArray; use crate::bitpack_decompress; use crate::unpack_iter::UnpackStrategy; @@ -92,13 +93,13 @@ pub(crate) fn fused_decompress< let strategy = FoRStrategy { reference: ref_ }; // Create [`UnpackedChunks`] with FoR strategy. - let mut unpacked = UnpackedChunks::new_with_strategy( + let mut unpacked = UnpackedChunks::try_new_with_strategy( strategy, bp.packed().as_host().clone(), bp.bit_width() as usize, bp.offset() as usize, bp.len(), - ); + )?; let mut builder = PrimitiveBuilder::::with_capacity( for_.reference_scalar().dtype().nullability(), @@ -107,7 +108,7 @@ pub(crate) fn fused_decompress< let mut uninit_range = builder.uninit_range(bp.len()); unsafe { // Append a dense null Mask. - uninit_range.append_mask(bp.validity_mask(bp.len(), bp.dtype().nullability())); + uninit_range.append_mask(bp.validity_mask()); } // SAFETY: `decode_into` will initialize all values in this range. @@ -116,7 +117,7 @@ pub(crate) fn fused_decompress< // Decode all chunks (initial, full, and trailer) in one call. unpacked.decode_into(uninit_slice); - if let Some(ref patches) = bp.patches(bp.len()) { + if let Some(ref patches) = bp.patches() { bitpack_decompress::apply_patches_to_uninit_range_fn( &mut uninit_range, patches, diff --git a/encodings/fastlanes/src/for/vtable/mod.rs b/encodings/fastlanes/src/for/vtable/mod.rs index 5b6c4f199b6..c6da24a5354 100644 --- a/encodings/fastlanes/src/for/vtable/mod.rs +++ b/encodings/fastlanes/src/for/vtable/mod.rs @@ -172,7 +172,7 @@ impl FoR { let reference = reference.cast(&dtype)?; let len = encoded.len(); let data = FoRData::try_new(encoded, reference)?; - Array::try_from_parts(ArrayParts::new(FoR, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(FoR, dtype, len, data)) }) } /// Encode a primitive array using Frame of Reference encoding. diff --git a/encodings/fastlanes/src/rle/vtable/mod.rs b/encodings/fastlanes/src/rle/vtable/mod.rs index c47f7adf5df..776fa1b84ad 100644 --- a/encodings/fastlanes/src/rle/vtable/mod.rs +++ b/encodings/fastlanes/src/rle/vtable/mod.rs @@ -23,7 +23,6 @@ use vortex_array::dtype::PType; use vortex_array::serde::ArrayChildren; use vortex_array::vtable; use vortex_array::vtable::VTable; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; @@ -216,7 +215,7 @@ impl RLE { ) -> VortexResult { let dtype = DType::Primitive(values.dtype().as_ptype(), indices.dtype().nullability()); let data = RLEData::try_new(values, indices, values_idx_offsets, offset, length)?; - Array::try_from_parts(ArrayParts::new(RLE, dtype, length, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(RLE, dtype, length, data)) }) } /// Create a new RLE array without validation. @@ -232,8 +231,7 @@ impl RLE { ) -> RLEArray { let dtype = DType::Primitive(values.dtype().as_ptype(), indices.dtype().nullability()); let data = unsafe { RLEData::new_unchecked(values, indices, values_idx_offsets, offset) }; - Array::try_from_parts(ArrayParts::new(RLE, dtype, length, data)) - .vortex_expect("pre-validated RLE parts must be valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(RLE, dtype, length, data)) } } /// Encode a primitive array using FastLanes RLE. diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index e267283a4cd..d716bc7608b 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -362,7 +362,7 @@ impl FSST { ) -> VortexResult { let len = codes.len(); let data = FSSTData::try_new(symbols, symbol_lengths, codes, uncompressed_lengths, &dtype)?; - Array::try_from_parts(ArrayParts::new(FSST, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(FSST, dtype, len, data)) }) } pub(crate) unsafe fn new_unchecked( @@ -376,8 +376,7 @@ impl FSST { let data = unsafe { FSSTData::new_unchecked(symbols, symbol_lengths, codes, uncompressed_lengths) }; - Array::try_from_parts(ArrayParts::new(FSST, dtype, len, data)) - .vortex_expect("pre-validated FSST parts must be valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(FSST, dtype, len, data)) } } } diff --git a/encodings/parquet-variant/src/array.rs b/encodings/parquet-variant/src/array.rs index 21ddf65f0f3..df872889391 100644 --- a/encodings/parquet-variant/src/array.rs +++ b/encodings/parquet-variant/src/array.rs @@ -80,7 +80,9 @@ impl ParquetVariant { let len = metadata.len(); let dtype = DType::Variant(validity.nullability()); let data = ParquetVariantData::try_new(validity, metadata, value, typed_value)?; - Array::try_from_parts(ArrayParts::new(ParquetVariant, dtype, len, data)) + Ok(unsafe { + Array::from_parts_unchecked(ArrayParts::new(ParquetVariant, dtype, len, data)) + }) } } diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index 93e19f2c9de..265566ec35e 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -281,7 +281,8 @@ impl Pco { pub(crate) fn try_new(dtype: DType, data: PcoData) -> VortexResult { let len = data.len(); - Array::try_from_parts(ArrayParts::new(Pco, dtype, len, data)) + data.validate(&dtype, len)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Pco, dtype, len, data)) }) } /// Compress a primitive array using pcodec. diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index 01f266a933f..a9fe74e7d36 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -215,8 +215,7 @@ impl RunEnd { ) -> RunEndArray { let data = unsafe { RunEndData::new_unchecked(ends, values, offset, length) }; let dtype = data.dtype().clone(); - Array::try_from_parts(ArrayParts::new(RunEnd, dtype, length, data)) - .vortex_expect("RunEndData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(RunEnd, dtype, length, data)) } } /// Build a new [`RunEndArray`] from ends and values. @@ -224,7 +223,7 @@ impl RunEnd { let len = RunEndData::logical_len_from_ends(&ends)?; let data = RunEndData::try_new_offset_length(ends, values, 0, len)?; let dtype = data.dtype().clone(); - Array::try_from_parts(ArrayParts::new(RunEnd, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(RunEnd, dtype, len, data)) }) } /// Build a new [`RunEndArray`] from ends, values, offset, and length. @@ -236,7 +235,7 @@ impl RunEnd { ) -> VortexResult { let data = RunEndData::try_new_offset_length(ends, values, offset, length)?; let dtype = data.dtype().clone(); - Array::try_from_parts(ArrayParts::new(RunEnd, dtype, length, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(RunEnd, dtype, length, data)) }) } /// Build a new [`RunEndArray`] from ends and values (panics on invalid input). @@ -249,7 +248,7 @@ impl RunEnd { let len = array.len(); let data = RunEndData::encode(array)?; let dtype = data.dtype().clone(); - Array::try_from_parts(ArrayParts::new(RunEnd, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(RunEnd, dtype, len, data)) }) } } diff --git a/encodings/sequence/src/array.rs b/encodings/sequence/src/array.rs index 8ad97e91f76..a3d50a420e1 100644 --- a/encodings/sequence/src/array.rs +++ b/encodings/sequence/src/array.rs @@ -408,9 +408,8 @@ impl Sequence { .vortex_expect("SequenceArray parts must be normalized to the target ptype"); let stats = Self::stats(multiplier); let data = unsafe { SequenceData::new_unchecked(base, multiplier) }; - Array::try_from_parts(ArrayParts::new(Sequence, dtype, length, data)) - .map(|array| array.with_stats_set(stats)) - .vortex_expect("pre-validated SequenceArray parts must be valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Sequence, dtype, length, data)) } + .with_stats_set(stats) } /// Construct a new [`SequenceArray`] from its components. @@ -424,8 +423,10 @@ impl Sequence { let dtype = DType::Primitive(ptype, nullability); let data = SequenceData::try_new(base, multiplier, ptype, nullability, length)?; let stats = Self::stats(data.multiplier()); - Array::try_from_parts(ArrayParts::new(Sequence, dtype, length, data)) - .map(|array| array.with_stats_set(stats)) + Ok( + unsafe { Array::from_parts_unchecked(ArrayParts::new(Sequence, dtype, length, data)) } + .with_stats_set(stats), + ) } /// Construct a new typed [`SequenceArray`] from base/multiplier values. @@ -439,8 +440,10 @@ impl Sequence { let dtype = DType::Primitive(ptype, nullability); let data = SequenceData::try_new_typed(base, multiplier, nullability, length)?; let stats = Self::stats(data.multiplier()); - Array::try_from_parts(ArrayParts::new(Sequence, dtype, length, data)) - .map(|array| array.with_stats_set(stats)) + Ok( + unsafe { Array::from_parts_unchecked(ArrayParts::new(Sequence, dtype, length, data)) } + .with_stats_set(stats), + ) } } diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index c192a6a0667..69b9c4e544e 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -253,22 +253,21 @@ impl Sparse { ) -> VortexResult { let dtype = fill_value.dtype().clone(); let data = SparseData::try_new(indices, values, len, fill_value)?; - Array::try_from_parts(ArrayParts::new(Sparse, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Sparse, dtype, len, data)) }) } pub fn try_new_from_patches(patches: Patches, fill_value: Scalar) -> VortexResult { let dtype = fill_value.dtype().clone(); let len = patches.array_len(); let data = SparseData::try_new_from_patches(patches, fill_value)?; - Array::try_from_parts(ArrayParts::new(Sparse, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Sparse, dtype, len, data)) }) } pub(crate) unsafe fn new_unchecked(patches: Patches, fill_value: Scalar) -> SparseArray { let dtype = fill_value.dtype().clone(); let len = patches.array_len(); let data = unsafe { SparseData::new_unchecked(patches, fill_value) }; - Array::try_from_parts(ArrayParts::new(Sparse, dtype, len, data)) - .vortex_expect("pre-validated SparseArray parts must be valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Sparse, dtype, len, data)) } } /// Encode the given array as a [`SparseArray`]. diff --git a/encodings/zigzag/src/array.rs b/encodings/zigzag/src/array.rs index df8b0195d72..896afd67fb3 100644 --- a/encodings/zigzag/src/array.rs +++ b/encodings/zigzag/src/array.rs @@ -176,7 +176,7 @@ impl ZigZag { let data = ZigZagData::try_new(encoded)?; let dtype = ZigZagData::dtype_from_encoded_dtype(data.encoded().dtype())?; let len = data.len(); - Array::try_from_parts(ArrayParts::new(ZigZag, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(ZigZag, dtype, len, data)) }) } } diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index 6c4929aa40e..a494cfc572f 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -265,7 +265,8 @@ impl Zstd { pub fn try_new(dtype: DType, data: ZstdData) -> VortexResult { let len = data.len(); - Array::try_from_parts(ArrayParts::new(Zstd, dtype, len, data)) + data.validate(&dtype, len)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Zstd, dtype, len, data)) }) } /// Compress a [`VarBinViewArray`] using Zstd without a dictionary. diff --git a/vortex-array/src/array/typed.rs b/vortex-array/src/array/typed.rs index aadd2485c94..7918c6b9956 100644 --- a/vortex-array/src/array/typed.rs +++ b/vortex-array/src/array/typed.rs @@ -178,7 +178,8 @@ impl Array { /// /// # Safety /// Caller must ensure the provided parts are logically consistent. - pub(crate) unsafe fn from_parts_unchecked(new: ArrayParts) -> Self { + #[doc(hidden)] + pub unsafe fn from_parts_unchecked(new: ArrayParts) -> Self { let inner = ArrayRef::from_inner(Arc::new(unsafe { ArrayInner::::from_data_unchecked( new.vtable, diff --git a/vortex-array/src/arrays/bool/array.rs b/vortex-array/src/arrays/bool/array.rs index 29eb9f1d0f2..688e5301c43 100644 --- a/vortex-array/src/arrays/bool/array.rs +++ b/vortex-array/src/arrays/bool/array.rs @@ -189,7 +189,7 @@ impl Array { let dtype = DType::Bool(validity.nullability()); let len = bits.len(); let data = BoolData::try_new(bits, validity)?; - Array::try_from_parts(ArrayParts::new(Bool, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Bool, dtype, len, data)) }) } /// Build a new bool array from a `BufferHandle`, returning an error if the offset is @@ -202,7 +202,7 @@ impl Array { ) -> VortexResult { let dtype = DType::Bool(validity.nullability()); let data = BoolData::try_new_from_handle(bits, offset, len, validity)?; - Array::try_from_parts(ArrayParts::new(Bool, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Bool, dtype, len, data)) }) } /// Creates a new [`BoolArray`] without validation. @@ -215,8 +215,7 @@ impl Array { let len = bits.len(); // SAFETY: caller guarantees validity length equals bit buffer length. let data = unsafe { BoolData::new_unchecked(bits, validity) }; - Array::try_from_parts(ArrayParts::new(Bool, dtype, len, data)) - .vortex_expect("BoolData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Bool, dtype, len, data)) } } /// Validates the components that would be used to create a [`BoolArray`]. diff --git a/vortex-array/src/arrays/chunked/array.rs b/vortex-array/src/arrays/chunked/array.rs index f3b3a9fc24c..33b471e1852 100644 --- a/vortex-array/src/arrays/chunked/array.rs +++ b/vortex-array/src/arrays/chunked/array.rs @@ -48,13 +48,14 @@ impl ChunkedData { ) -> Vec> { let mut slots = Vec::with_capacity(1 + chunks.len()); slots.push(Some( - Array::try_from_parts(ArrayParts::new( - crate::arrays::Primitive, - chunk_offsets.dtype(), - chunk_offsets.len(), - chunk_offsets.clone(), - )) - .vortex_expect("chunk offsets are always valid") + unsafe { + Array::from_parts_unchecked(ArrayParts::new( + crate::arrays::Primitive, + chunk_offsets.dtype(), + chunk_offsets.len(), + chunk_offsets.clone(), + )) + } .into_array(), )); slots.extend(chunks.iter().map(|c| Some(c.clone()))); @@ -275,17 +276,19 @@ impl Array { let data = ChunkedData::try_new(chunks, dtype)?; let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayParts::new(Chunked, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Chunked, dtype, len, data)) }) } pub fn rechunk(&self, target_bytesize: u64, target_rowsize: usize) -> VortexResult { let data = self.data().rechunk(target_bytesize, target_rowsize)?; - Array::try_from_parts(ArrayParts::new( - Chunked, - self.dtype().clone(), - data.len(), - data, - )) + Ok(unsafe { + Array::from_parts_unchecked(ArrayParts::new( + Chunked, + self.dtype().clone(), + data.len(), + data, + )) + }) } /// Creates a new `ChunkedArray` without validation. @@ -297,8 +300,7 @@ impl Array { let data = unsafe { ChunkedData::new_unchecked(chunks, dtype) }; let dtype = data.dtype().clone(); let len = data.len(); - Array::try_from_parts(ArrayParts::new(Chunked, dtype, len, data)) - .vortex_expect("ChunkedData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Chunked, dtype, len, data)) } } } diff --git a/vortex-array/src/arrays/constant/array.rs b/vortex-array/src/arrays/constant/array.rs index 5514221d379..3841a9d2b02 100644 --- a/vortex-array/src/arrays/constant/array.rs +++ b/vortex-array/src/arrays/constant/array.rs @@ -1,8 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use vortex_error::VortexExpect; - use crate::ArrayRef; use crate::array::Array; use crate::array::ArrayParts; @@ -47,7 +45,6 @@ impl Array { let scalar = scalar.into(); let dtype = scalar.dtype().clone(); let data = ConstantData::new(scalar); - Array::try_from_parts(ArrayParts::new(Constant, dtype, len, data)) - .vortex_expect("ConstantData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Constant, dtype, len, data)) } } } diff --git a/vortex-array/src/arrays/decimal/array.rs b/vortex-array/src/arrays/decimal/array.rs index a798d5d26bc..a51657fd958 100644 --- a/vortex-array/src/arrays/decimal/array.rs +++ b/vortex-array/src/arrays/decimal/array.rs @@ -448,8 +448,7 @@ impl Array { let dtype = DType::Decimal(decimal_dtype, validity.nullability()); let len = buffer.len(); let data = DecimalData::new(buffer, decimal_dtype, validity); - Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data)) - .vortex_expect("DecimalData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } } /// Creates a new [`DecimalArray`] without validation. @@ -465,8 +464,7 @@ impl Array { let dtype = DType::Decimal(decimal_dtype, validity.nullability()); let len = buffer.len(); let data = unsafe { DecimalData::new_unchecked(buffer, decimal_dtype, validity) }; - Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data)) - .vortex_expect("DecimalData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } } /// Creates a new [`DecimalArray`] from a host-native buffer with validation. @@ -478,7 +476,7 @@ impl Array { let dtype = DType::Decimal(decimal_dtype, validity.nullability()); let len = buffer.len(); let data = DecimalData::try_new(buffer, decimal_dtype, validity)?; - Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) }) } /// Creates a new [`DecimalArray`] from an iterator of values. @@ -493,8 +491,7 @@ impl Array { let data = DecimalData::from_iter(iter, decimal_dtype); let dtype = data.dtype(); let len = data.len(); - Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data)) - .vortex_expect("DecimalData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } } /// Creates a new [`DecimalArray`] from an iterator of optional values. @@ -505,8 +502,7 @@ impl Array { let data = DecimalData::from_option_iter(iter, decimal_dtype); let dtype = data.dtype(); let len = data.len(); - Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data)) - .vortex_expect("DecimalData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } } /// Creates a new [`DecimalArray`] from a [`BufferHandle`]. @@ -519,8 +515,7 @@ impl Array { let dtype = DType::Decimal(decimal_dtype, validity.nullability()); let len = values.len() / values_type.byte_width(); let data = DecimalData::new_handle(values, values_type, decimal_dtype, validity); - Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data)) - .vortex_expect("DecimalData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } } /// Creates a new [`DecimalArray`] without validation from a [`BufferHandle`]. @@ -539,8 +534,7 @@ impl Array { let data = unsafe { DecimalData::new_unchecked_handle(values, values_type, decimal_dtype, validity) }; - Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data)) - .vortex_expect("DecimalData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } } } diff --git a/vortex-array/src/arrays/dict/array.rs b/vortex-array/src/arrays/dict/array.rs index c452983f62c..a791d79b798 100644 --- a/vortex-array/src/arrays/dict/array.rs +++ b/vortex-array/src/arrays/dict/array.rs @@ -200,8 +200,7 @@ impl Array { let data = DictData::new(codes, values); let dtype = data.dtype(); let len = data.len(); - Array::try_from_parts(ArrayParts::new(Dict, dtype, len, data)) - .vortex_expect("DictData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Dict, dtype, len, data)) } } /// Build a new `DictArray` from its components, `codes` and `values`. @@ -209,7 +208,7 @@ impl Array { let data = DictData::try_new(codes, values)?; let dtype = data.dtype(); let len = data.len(); - Array::try_from_parts(ArrayParts::new(Dict, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Dict, dtype, len, data)) }) } /// Build a new `DictArray` without validating the codes or values. @@ -221,8 +220,7 @@ impl Array { let data = unsafe { DictData::new_unchecked(codes, values) }; let dtype = data.dtype(); let len = data.len(); - Array::try_from_parts(ArrayParts::new(Dict, dtype, len, data)) - .vortex_expect("DictData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Dict, dtype, len, data)) } } /// Set whether all values in the dictionary are referenced by at least one code. @@ -237,8 +235,7 @@ impl Array { self.into_data() .set_all_values_referenced(all_values_referenced) }; - Array::try_from_parts(ArrayParts::new(Dict, dtype, len, data)) - .vortex_expect("data is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Dict, dtype, len, data)) } } } diff --git a/vortex-array/src/arrays/extension/array.rs b/vortex-array/src/arrays/extension/array.rs index d20d53e0e56..757c3042484 100644 --- a/vortex-array/src/arrays/extension/array.rs +++ b/vortex-array/src/arrays/extension/array.rs @@ -151,8 +151,7 @@ impl Array { let dtype = DType::Extension(ext_dtype.clone()); let len = storage_array.len(); let data = ExtensionData::new(ext_dtype, storage_array); - Array::try_from_parts(ArrayParts::new(Extension, dtype, len, data)) - .vortex_expect("ExtensionData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Extension, dtype, len, data)) } } /// Tries to construct a new `ExtensionArray`. @@ -160,6 +159,6 @@ impl Array { let dtype = DType::Extension(ext_dtype.clone()); let len = storage_array.len(); let data = ExtensionData::try_new(ext_dtype, storage_array)?; - Array::try_from_parts(ArrayParts::new(Extension, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Extension, dtype, len, data)) }) } } diff --git a/vortex-array/src/arrays/filter/array.rs b/vortex-array/src/arrays/filter/array.rs index 6ff5adfa313..7a0677d5d92 100644 --- a/vortex-array/src/arrays/filter/array.rs +++ b/vortex-array/src/arrays/filter/array.rs @@ -98,8 +98,7 @@ impl Array { let dtype = array.dtype().clone(); let len = mask.true_count(); let data = FilterData::new(array, mask); - Array::try_from_parts(ArrayParts::new(Filter, dtype, len, data)) - .vortex_expect("FilterData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Filter, dtype, len, data)) } } /// Constructs a new `FilterArray`. @@ -107,6 +106,6 @@ impl Array { let dtype = array.dtype().clone(); let len = mask.true_count(); let data = FilterData::try_new(array, mask)?; - Array::try_from_parts(ArrayParts::new(Filter, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Filter, dtype, len, data)) }) } } diff --git a/vortex-array/src/arrays/fixed_size_list/array.rs b/vortex-array/src/arrays/fixed_size_list/array.rs index eb186d4d583..eb1ef6d7aff 100644 --- a/vortex-array/src/arrays/fixed_size_list/array.rs +++ b/vortex-array/src/arrays/fixed_size_list/array.rs @@ -275,8 +275,7 @@ impl Array { validity.nullability(), ); let data = FixedSizeListData::new(elements, list_size, validity, len); - Array::try_from_parts(ArrayParts::new(FixedSizeList, dtype, len, data)) - .vortex_expect("FixedSizeListData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(FixedSizeList, dtype, len, data)) } } /// Constructs a new `FixedSizeListArray`. @@ -292,7 +291,11 @@ impl Array { validity.nullability(), ); let data = FixedSizeListData::try_new(elements, list_size, validity, len)?; - Array::try_from_parts(ArrayParts::new(FixedSizeList, dtype, len, data)) + Ok( + unsafe { + Array::from_parts_unchecked(ArrayParts::new(FixedSizeList, dtype, len, data)) + }, + ) } /// Creates a new `FixedSizeListArray` without validation. @@ -312,8 +315,7 @@ impl Array { validity.nullability(), ); let data = unsafe { FixedSizeListData::new_unchecked(elements, list_size, validity, len) }; - Array::try_from_parts(ArrayParts::new(FixedSizeList, dtype, len, data)) - .vortex_expect("FixedSizeListData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(FixedSizeList, dtype, len, data)) } } } diff --git a/vortex-array/src/arrays/list/array.rs b/vortex-array/src/arrays/list/array.rs index 631fe09d56e..b8f99c35804 100644 --- a/vortex-array/src/arrays/list/array.rs +++ b/vortex-array/src/arrays/list/array.rs @@ -361,8 +361,7 @@ impl Array { let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); let len = offsets.len().saturating_sub(1); let data = ListData::new(elements, offsets, validity); - Array::try_from_parts(ArrayParts::new(List, dtype, len, data)) - .vortex_expect("ListData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) } } /// Constructs a new `ListArray`. @@ -374,7 +373,7 @@ impl Array { let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); let len = offsets.len().saturating_sub(1); let data = ListData::try_new(elements, offsets, validity)?; - Array::try_from_parts(ArrayParts::new(List, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) }) } /// Creates a new `ListArray` without validation. @@ -386,8 +385,7 @@ impl Array { let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); let len = offsets.len().saturating_sub(1); let data = unsafe { ListData::new_unchecked(elements, offsets, validity) }; - Array::try_from_parts(ArrayParts::new(List, dtype, len, data)) - .vortex_expect("ListData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) } } } @@ -400,7 +398,9 @@ impl ListData { let data = child_list_array.reset_offsets(recurse)?; let dtype = data.dtype(); let len = data.len(); - elements = Array::try_from_parts(ArrayParts::new(List, dtype, len, data))?.into_array(); + elements = + unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) } + .into_array(); } let offsets = self.offsets(); diff --git a/vortex-array/src/arrays/listview/array.rs b/vortex-array/src/arrays/listview/array.rs index fa7d3941657..c25dce40f43 100644 --- a/vortex-array/src/arrays/listview/array.rs +++ b/vortex-array/src/arrays/listview/array.rs @@ -481,8 +481,7 @@ impl Array { let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); let len = offsets.len(); let data = ListViewData::new(elements, offsets, sizes, validity); - Array::try_from_parts(ArrayParts::new(ListView, dtype, len, data)) - .vortex_expect("ListViewData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(ListView, dtype, len, data)) } } /// Constructs a new `ListViewArray`. @@ -495,7 +494,7 @@ impl Array { let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); let len = offsets.len(); let data = ListViewData::try_new(elements, offsets, sizes, validity)?; - Array::try_from_parts(ArrayParts::new(ListView, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(ListView, dtype, len, data)) }) } /// Creates a new `ListViewArray` without validation. @@ -512,8 +511,7 @@ impl Array { let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); let len = offsets.len(); let data = unsafe { ListViewData::new_unchecked(elements, offsets, sizes, validity) }; - Array::try_from_parts(ArrayParts::new(ListView, dtype, len, data)) - .vortex_expect("ListViewData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(ListView, dtype, len, data)) } } /// Mark whether this list view can be zero-copy converted to a list. @@ -525,8 +523,7 @@ impl Array { let dtype = self.dtype().clone(); let len = self.len(); let data = unsafe { self.into_data().with_zero_copy_to_list(is_zctl) }; - Array::try_from_parts(ArrayParts::new(ListView, dtype, len, data)) - .vortex_expect("data is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(ListView, dtype, len, data)) } } } diff --git a/vortex-array/src/arrays/listview/conversion.rs b/vortex-array/src/arrays/listview/conversion.rs index 550c745d499..d2334c59526 100644 --- a/vortex-array/src/arrays/listview/conversion.rs +++ b/vortex-array/src/arrays/listview/conversion.rs @@ -41,7 +41,8 @@ pub fn list_view_from_list(list: ListArray, ctx: &mut ExecutionCtx) -> VortexRes let data = list.reset_offsets(false).vortex_expect("This can't fail"); let dtype = data.dtype(); let len = data.len(); - let list: ListArray = Array::try_from_parts(ArrayParts::new(List, dtype, len, data))?; + let list: ListArray = + unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) }; let list_offsets = list.offsets().clone(); diff --git a/vortex-array/src/arrays/masked/array.rs b/vortex-array/src/arrays/masked/array.rs index c395dedeed2..9e011140930 100644 --- a/vortex-array/src/arrays/masked/array.rs +++ b/vortex-array/src/arrays/masked/array.rs @@ -92,6 +92,6 @@ impl Array { let dtype = child.dtype().as_nullable(); let len = child.len(); let data = MaskedData::try_new(child, validity)?; - Array::try_from_parts(ArrayParts::new(Masked, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Masked, dtype, len, data)) }) } } diff --git a/vortex-array/src/arrays/null/mod.rs b/vortex-array/src/arrays/null/mod.rs index 292a7c8ab4f..183a2008654 100644 --- a/vortex-array/src/arrays/null/mod.rs +++ b/vortex-array/src/arrays/null/mod.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; @@ -162,8 +161,9 @@ impl Null { impl Array { pub fn new(len: usize) -> Self { - Array::try_from_parts(ArrayParts::new(Null, DType::Null, len, NullData::new())) - .vortex_expect("NullData is always valid") + unsafe { + Array::from_parts_unchecked(ArrayParts::new(Null, DType::Null, len, NullData::new())) + } } } diff --git a/vortex-array/src/arrays/patched/array.rs b/vortex-array/src/arrays/patched/array.rs index f5470aac6df..4976a103dd6 100644 --- a/vortex-array/src/arrays/patched/array.rs +++ b/vortex-array/src/arrays/patched/array.rs @@ -132,9 +132,10 @@ impl IntoArray for PatchedArray { fn into_array(self) -> ArrayRef { let dtype = self.base_array().dtype().clone(); let len = self.len(); - Array::::try_from_parts(ArrayParts::new(Patched, dtype, len, self)) - .vortex_expect("PatchedArray is always valid") - .into_array() + unsafe { + Array::::from_parts_unchecked(ArrayParts::new(Patched, dtype, len, self)) + } + .into_array() } } diff --git a/vortex-array/src/arrays/primitive/array/mod.rs b/vortex-array/src/arrays/primitive/array/mod.rs index 0675c50d9f1..25cbf8c755b 100644 --- a/vortex-array/src/arrays/primitive/array/mod.rs +++ b/vortex-array/src/arrays/primitive/array/mod.rs @@ -198,8 +198,7 @@ impl Array { let dtype = DType::Primitive(T::PTYPE, nullability); let len = 0; let data = PrimitiveData::empty::(nullability); - Array::try_from_parts(ArrayParts::new(Primitive, dtype, len, data)) - .vortex_expect("PrimitiveData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } } /// Creates a new `PrimitiveArray`. @@ -212,8 +211,7 @@ impl Array { let dtype = DType::Primitive(T::PTYPE, validity.nullability()); let len = buffer.len(); let data = PrimitiveData::new(buffer, validity); - Array::try_from_parts(ArrayParts::new(Primitive, dtype, len, data)) - .vortex_expect("PrimitiveData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } } /// Constructs a new `PrimitiveArray`. @@ -221,7 +219,7 @@ impl Array { let dtype = DType::Primitive(T::PTYPE, validity.nullability()); let len = buffer.len(); let data = PrimitiveData::try_new(buffer, validity)?; - Array::try_from_parts(ArrayParts::new(Primitive, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) }) } /// Creates a new `PrimitiveArray` without validation. @@ -233,8 +231,7 @@ impl Array { let dtype = DType::Primitive(T::PTYPE, validity.nullability()); let len = buffer.len(); let data = unsafe { PrimitiveData::new_unchecked(buffer, validity) }; - Array::try_from_parts(ArrayParts::new(Primitive, dtype, len, data)) - .vortex_expect("PrimitiveData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } } /// Create a new array from a buffer handle. @@ -250,8 +247,7 @@ impl Array { let dtype = DType::Primitive(ptype, validity.nullability()); let len = handle.len() / ptype.byte_width(); let data = unsafe { PrimitiveData::new_unchecked_from_handle(handle, ptype, validity) }; - Array::try_from_parts(ArrayParts::new(Primitive, dtype, len, data)) - .vortex_expect("PrimitiveData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } } /// Creates a new `PrimitiveArray` from a [`BufferHandle`]. @@ -268,8 +264,7 @@ impl Array { let dtype = DType::Primitive(ptype, validity.nullability()); let len = buffer.len() / ptype.byte_width(); let data = PrimitiveData::from_byte_buffer(buffer, ptype, validity); - Array::try_from_parts(ArrayParts::new(Primitive, dtype, len, data)) - .vortex_expect("PrimitiveData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } } /// Create a PrimitiveArray from a byte buffer containing only the valid elements. @@ -283,8 +278,7 @@ impl Array { let len = n_rows; let data = PrimitiveData::from_values_byte_buffer(valid_elems_buffer, ptype, validity, n_rows); - Array::try_from_parts(ArrayParts::new(Primitive, dtype, len, data)) - .vortex_expect("PrimitiveData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } } /// Validates the components that would be used to create a `PrimitiveArray`. diff --git a/vortex-array/src/arrays/scalar_fn/array.rs b/vortex-array/src/arrays/scalar_fn/array.rs index 4c394ddf009..1fc9accc886 100644 --- a/vortex-array/src/arrays/scalar_fn/array.rs +++ b/vortex-array/src/arrays/scalar_fn/array.rs @@ -92,6 +92,6 @@ impl Array { let dtype = scalar_fn.return_dtype(&arg_dtypes)?; let data = ScalarFnData::try_new(scalar_fn.clone(), children, len)?; let vtable = ScalarFnVTable { scalar_fn }; - Array::try_from_parts(ArrayParts::new(vtable, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(vtable, dtype, len, data)) }) } } diff --git a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs index f3fbbe52894..c74a53d7730 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs @@ -196,7 +196,10 @@ pub trait ScalarFnArrayExt: scalar_fn::ScalarFnVTable { slots: children.into_iter().map(Some).collect(), }; let vtable = ScalarFnVTable { scalar_fn }; - Array::try_from_parts(ArrayParts::new(vtable, dtype, len, data)).map(IntoArray::into_array) + Ok( + unsafe { Array::from_parts_unchecked(ArrayParts::new(vtable, dtype, len, data)) } + .into_array(), + ) } } impl ScalarFnArrayExt for V {} diff --git a/vortex-array/src/arrays/shared/array.rs b/vortex-array/src/arrays/shared/array.rs index 712a1c2a479..cf4df660520 100644 --- a/vortex-array/src/arrays/shared/array.rs +++ b/vortex-array/src/arrays/shared/array.rs @@ -123,8 +123,14 @@ impl Array { pub fn new(source: ArrayRef) -> Self { let dtype = source.dtype().clone(); let len = source.len(); - Array::try_from_parts(ArrayParts::new(Shared, dtype, len, SharedData::new(source))) - .vortex_expect("SharedData is always valid") + unsafe { + Array::from_parts_unchecked(ArrayParts::new( + Shared, + dtype, + len, + SharedData::new(source), + )) + } } } diff --git a/vortex-array/src/arrays/slice/array.rs b/vortex-array/src/arrays/slice/array.rs index 1292bcfbc01..2d438c3fe9c 100644 --- a/vortex-array/src/arrays/slice/array.rs +++ b/vortex-array/src/arrays/slice/array.rs @@ -91,7 +91,7 @@ impl Array { let len = range.len(); let dtype = child.dtype().clone(); let data = SliceData::try_new(child, range)?; - Array::try_from_parts(ArrayParts::new(Slice, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Slice, dtype, len, data)) }) } /// Constructs a new `SliceArray`. @@ -99,7 +99,6 @@ impl Array { let len = range.len(); let dtype = child.dtype().clone(); let data = SliceData::new(child, range); - Array::try_from_parts(ArrayParts::new(Slice, dtype, len, data)) - .vortex_expect("SliceData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Slice, dtype, len, data)) } } } diff --git a/vortex-array/src/arrays/struct_/array.rs b/vortex-array/src/arrays/struct_/array.rs index 3758d0d75c6..7686bce9989 100644 --- a/vortex-array/src/arrays/struct_/array.rs +++ b/vortex-array/src/arrays/struct_/array.rs @@ -559,8 +559,7 @@ impl Array { let data = StructData::new(names, fields, length, validity); let dtype = data.dtype(); let len = data.len(); - Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) - .vortex_expect("StructData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) } } /// Constructs a new `StructArray`. @@ -573,7 +572,7 @@ impl Array { let data = StructData::try_new(names, fields, length, validity)?; let dtype = data.dtype(); let len = data.len(); - Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) } /// Creates a new `StructArray` without validation. @@ -590,8 +589,7 @@ impl Array { let data = unsafe { StructData::new_unchecked(fields, dtype, length, validity) }; let dtype = data.dtype(); let len = data.len(); - Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) - .vortex_expect("StructData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) } } /// Constructs a new `StructArray` with an explicit dtype. @@ -604,7 +602,7 @@ impl Array { let data = StructData::try_new_with_dtype(fields, dtype, length, validity)?; let dtype = data.dtype(); let len = data.len(); - Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) } /// Construct a `StructArray` from named fields. @@ -612,7 +610,7 @@ impl Array { let data = StructData::from_fields(items)?; let dtype = data.dtype(); let len = data.len(); - Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) } /// Create a `StructArray` from an iterator of (name, array) pairs with validity. @@ -627,7 +625,7 @@ impl Array { let data = StructData::try_from_iter_with_validity(iter, validity)?; let dtype = data.dtype(); let len = data.len(); - Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) } /// Create a `StructArray` from an iterator of (name, array) pairs. @@ -637,7 +635,7 @@ impl Array { let data = StructData::try_from_iter(iter)?; let dtype = data.dtype(); let len = data.len(); - Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) } /// Create a fieldless `StructArray` with the given length. @@ -645,8 +643,7 @@ impl Array { let data = StructData::new_fieldless_with_len(len); let dtype = data.dtype(); let len = data.len(); - Array::try_from_parts(ArrayParts::new(Struct, dtype, len, data)) - .vortex_expect("StructData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) } } } diff --git a/vortex-array/src/arrays/varbin/array.rs b/vortex-array/src/arrays/varbin/array.rs index f91e1d5e9b4..329cf3da04c 100644 --- a/vortex-array/src/arrays/varbin/array.rs +++ b/vortex-array/src/arrays/varbin/array.rs @@ -400,12 +400,15 @@ impl VarBinData { /// Forwarding constructors for `VarBinArray` (= `Array`). impl Array { - pub fn from_vec>(vec: Vec, dtype: DType) -> Self { - let data = VarBinData::from_vec(vec, dtype); + #[inline] + fn from_prevalidated_data(data: VarBinData) -> Self { let dtype = data.dtype(); let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) - .vortex_expect("VarBinData is always valid") + unsafe { Array::from_parts_unchecked(ArrayParts::new(VarBin, dtype, len, data)) } + } + + pub fn from_vec>(vec: Vec, dtype: DType) -> Self { + Self::from_prevalidated_data(VarBinData::from_vec(vec, dtype)) } #[expect( @@ -416,22 +419,14 @@ impl Array { iter: I, dtype: DType, ) -> Self { - let data = VarBinData::from_iter(iter, dtype); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) - .vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from_iter(iter, dtype)) } pub fn from_iter_nonnull, I: IntoIterator>( iter: I, dtype: DType, ) -> Self { - let data = VarBinData::from_iter_nonnull(iter, dtype); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) - .vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from_iter_nonnull(iter, dtype)) } /// Create from a vector of string slices. @@ -500,11 +495,7 @@ impl VarBinData { impl Array { /// Creates a new `VarBinArray`. pub fn new(offsets: ArrayRef, bytes: ByteBuffer, dtype: DType, validity: Validity) -> Self { - let data = VarBinData::new(offsets, bytes, dtype, validity); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) - .vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::new(offsets, bytes, dtype, validity)) } /// Creates a new `VarBinArray` without validation. @@ -518,11 +509,9 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - let data = unsafe { VarBinData::new_unchecked(offsets, bytes, dtype, validity) }; - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) - .vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(unsafe { + VarBinData::new_unchecked(offsets, bytes, dtype, validity) + }) } /// Creates a new `VarBinArray` without validation from a [`BufferHandle`]. @@ -536,12 +525,9 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - let data = - unsafe { VarBinData::new_unchecked_from_handle(offsets, bytes, dtype, validity) }; - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) - .vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(unsafe { + VarBinData::new_unchecked_from_handle(offsets, bytes, dtype, validity) + }) } /// Constructs a new `VarBinArray`. @@ -551,10 +537,9 @@ impl Array { dtype: DType, validity: Validity, ) -> VortexResult { - let data = VarBinData::try_new(offsets, bytes, dtype, validity)?; - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) + Ok(Self::from_prevalidated_data(VarBinData::try_new( + offsets, bytes, dtype, validity, + )?)) } } @@ -634,120 +619,72 @@ impl<'a> FromIterator> for VarBinData { impl From> for Array { fn from(value: Vec<&[u8]>) -> Self { - let data = VarBinData::from(value); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) - .vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from(value)) } } impl From>> for Array { fn from(value: Vec>) -> Self { - let data = VarBinData::from(value); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) - .vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from(value)) } } impl From> for Array { fn from(value: Vec) -> Self { - let data = VarBinData::from(value); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) - .vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from(value)) } } impl From> for Array { fn from(value: Vec<&str>) -> Self { - let data = VarBinData::from(value); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) - .vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from(value)) } } impl From>> for Array { fn from(value: Vec>) -> Self { - let data = VarBinData::from(value); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) - .vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from(value)) } } impl From>>> for Array { fn from(value: Vec>>) -> Self { - let data = VarBinData::from(value); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) - .vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from(value)) } } impl From>> for Array { fn from(value: Vec>) -> Self { - let data = VarBinData::from(value); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) - .vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from(value)) } } impl From>> for Array { fn from(value: Vec>) -> Self { - let data = VarBinData::from(value); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) - .vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from(value)) } } impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - let data = >::from_iter(iter); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) - .vortex_expect(" is always valid") + Self::from_prevalidated_data(>::from_iter(iter)) } } impl FromIterator>> for Array { fn from_iter>>>(iter: T) -> Self { - let data = >::from_iter(iter); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) - .vortex_expect(" is always valid") + Self::from_prevalidated_data(>::from_iter(iter)) } } impl FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - let data = >::from_iter(iter); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) - .vortex_expect(" is always valid") + Self::from_prevalidated_data(>::from_iter(iter)) } } impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - let data = >::from_iter(iter); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBin, dtype, len, data)) - .vortex_expect(" is always valid") + Self::from_prevalidated_data(>::from_iter(iter)) } } diff --git a/vortex-array/src/arrays/varbinview/array.rs b/vortex-array/src/arrays/varbinview/array.rs index 4a0e9febbe7..fd61c204feb 100644 --- a/vortex-array/src/arrays/varbinview/array.rs +++ b/vortex-array/src/arrays/varbinview/array.rs @@ -570,6 +570,13 @@ impl VarBinViewData { } impl Array { + #[inline] + fn from_prevalidated_data(data: VarBinViewData) -> Self { + let dtype = data.dtype(); + let len = data.len(); + unsafe { Array::from_parts_unchecked(ArrayParts::new(VarBinView, dtype, len, data)) } + } + /// Construct a `VarBinViewArray` from an iterator of optional byte slices. #[expect( clippy::same_name_method, @@ -579,47 +586,27 @@ impl Array { iter: I, dtype: DType, ) -> Self { - let data = VarBinViewData::from_iter(iter, dtype); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) - .vortex_expect("VarBinViewData is always valid") + Self::from_prevalidated_data(VarBinViewData::from_iter(iter, dtype)) } pub fn from_iter_str, I: IntoIterator>(iter: I) -> Self { - let data = VarBinViewData::from_iter_str(iter); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) - .vortex_expect("VarBinViewData is always valid") + Self::from_prevalidated_data(VarBinViewData::from_iter_str(iter)) } pub fn from_iter_nullable_str, I: IntoIterator>>( iter: I, ) -> Self { - let data = VarBinViewData::from_iter_nullable_str(iter); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) - .vortex_expect("VarBinViewData is always valid") + Self::from_prevalidated_data(VarBinViewData::from_iter_nullable_str(iter)) } pub fn from_iter_bin, I: IntoIterator>(iter: I) -> Self { - let data = VarBinViewData::from_iter_bin(iter); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) - .vortex_expect("VarBinViewData is always valid") + Self::from_prevalidated_data(VarBinViewData::from_iter_bin(iter)) } pub fn from_iter_nullable_bin, I: IntoIterator>>( iter: I, ) -> Self { - let data = VarBinViewData::from_iter_nullable_bin(iter); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) - .vortex_expect("VarBinViewData is always valid") + Self::from_prevalidated_data(VarBinViewData::from_iter_nullable_bin(iter)) } /// Creates a new `VarBinViewArray`. @@ -629,10 +616,9 @@ impl Array { dtype: DType, validity: Validity, ) -> VortexResult { - let data = VarBinViewData::try_new(views, buffers, dtype, validity)?; - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) + Ok(Self::from_prevalidated_data(VarBinViewData::try_new( + views, buffers, dtype, validity, + )?)) } /// Creates a new `VarBinViewArray` without validation. @@ -646,11 +632,9 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - let data = unsafe { VarBinViewData::new_unchecked(views, buffers, dtype, validity) }; - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) - .vortex_expect("VarBinViewData is always valid") + Self::from_prevalidated_data(unsafe { + VarBinViewData::new_unchecked(views, buffers, dtype, validity) + }) } /// Creates a new `VarBinViewArray` with device or host memory. @@ -660,11 +644,7 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - let data = VarBinViewData::new_handle(views, buffers, dtype, validity); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) - .vortex_expect("VarBinViewData is always valid") + Self::from_prevalidated_data(VarBinViewData::new_handle(views, buffers, dtype, validity)) } /// Construct a new array from `BufferHandle`s without validation. @@ -678,11 +658,9 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - let data = unsafe { VarBinViewData::new_handle_unchecked(views, buffers, dtype, validity) }; - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) - .vortex_expect("VarBinViewData is always valid") + Self::from_prevalidated_data(unsafe { + VarBinViewData::new_handle_unchecked(views, buffers, dtype, validity) + }) } } @@ -714,40 +692,24 @@ impl<'a> FromIterator> for VarBinViewData { impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - let data = >::from_iter(iter); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) - .vortex_expect(" is always valid") + Self::from_prevalidated_data(>::from_iter(iter)) } } impl FromIterator>> for Array { fn from_iter>>>(iter: T) -> Self { - let data = >::from_iter(iter); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) - .vortex_expect(" is always valid") + Self::from_prevalidated_data(>::from_iter(iter)) } } impl FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - let data = >::from_iter(iter); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) - .vortex_expect(" is always valid") + Self::from_prevalidated_data(>::from_iter(iter)) } } impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - let data = >::from_iter(iter); - let dtype = data.dtype(); - let len = data.len(); - Array::try_from_parts(ArrayParts::new(VarBinView, dtype, len, data)) - .vortex_expect(" is always valid") + Self::from_prevalidated_data(>::from_iter(iter)) } } diff --git a/vortex-array/src/arrays/variant/mod.rs b/vortex-array/src/arrays/variant/mod.rs index 1d9a60e51f3..02a89fae953 100644 --- a/vortex-array/src/arrays/variant/mod.rs +++ b/vortex-array/src/arrays/variant/mod.rs @@ -65,13 +65,14 @@ impl Array { let dtype = DType::Variant(child.dtype().nullability()); let len = child.len(); let stats = child.statistics().to_owned(); - Array::try_from_parts(ArrayParts::new( - Variant, - dtype, - len, - VariantData::new(child), - )) - .map(|array| array.with_stats_set(stats)) - .vortex_expect("VariantData is always valid") + unsafe { + Array::from_parts_unchecked(ArrayParts::new( + Variant, + dtype, + len, + VariantData::new(child), + )) + } + .with_stats_set(stats) } } From 2254f42ce8da23a15b143ea32e1670aee3644c33 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Fri, 3 Apr 2026 14:02:24 -0400 Subject: [PATCH 09/15] Remove triggers Signed-off-by: Nicholas Gates --- docs/api/python/arrays.rst | 2 +- .../fastlanes/src/for/array/for_compress.rs | 4 +- encodings/fastlanes/src/for/vtable/mod.rs | 13 +++++ encodings/sparse/src/lib.rs | 54 +++++++++++++++---- 4 files changed, 59 insertions(+), 14 deletions(-) diff --git a/docs/api/python/arrays.rst b/docs/api/python/arrays.rst index da6c6a7a449..55d4bc6d0b3 100644 --- a/docs/api/python/arrays.rst +++ b/docs/api/python/arrays.rst @@ -144,7 +144,7 @@ Registry and Serde .. autoclass:: vortex.ArrayContext :members: -.. autoclass:: vortex.ArrayParts +.. autoclass:: vortex.SerializedArray :members: diff --git a/encodings/fastlanes/src/for/array/for_compress.rs b/encodings/fastlanes/src/for/array/for_compress.rs index 6bd4b36703a..a1310083a4d 100644 --- a/encodings/fastlanes/src/for/array/for_compress.rs +++ b/encodings/fastlanes/src/for/array/for_compress.rs @@ -25,9 +25,7 @@ impl FoRData { let encoded = match_each_integer_ptype!(array.ptype(), |T| { compress_primitive::(array, T::try_from(&min)?)?.into_array() }); - let for_array = FoR::try_new(encoded, min)?; - for_array.statistics().inherit_from(array_ref.statistics()); - Ok(for_array) + Ok(FoR::try_new(encoded, min)?) } } diff --git a/encodings/fastlanes/src/for/vtable/mod.rs b/encodings/fastlanes/src/for/vtable/mod.rs index c6da24a5354..157645c410f 100644 --- a/encodings/fastlanes/src/for/vtable/mod.rs +++ b/encodings/fastlanes/src/for/vtable/mod.rs @@ -180,3 +180,16 @@ impl FoR { FoRData::encode(array) } } + +#[cfg(test)] +mod tests { + use vortex_array::scalar::ScalarValue; + use vortex_array::test_harness::check_metadata; + + #[cfg_attr(miri, ignore)] + #[test] + fn test_for_metadata() { + let metadata: Vec = ScalarValue::to_proto_bytes(Some(&ScalarValue::from(i64::MAX))); + check_metadata("for.metadata", &metadata); + } +} diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index 69b9c4e544e..fc9613a369d 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -277,6 +277,44 @@ impl Sparse { } impl SparseData { + fn normalize_values_dtype(values: ArrayRef, fill_value: &Scalar) -> VortexResult { + let fill_dtype = fill_value.dtype(); + let values_dtype = values.dtype(); + + vortex_ensure!( + values_dtype.eq_ignore_nullability(fill_dtype), + "fill value, {:?}, should be instance of values dtype, {} but was {}.", + fill_value, + values_dtype, + fill_dtype, + ); + + if values_dtype == fill_dtype { + Ok(values) + } else { + values.cast(fill_dtype.clone()) + } + } + + fn normalize_patches_dtype(patches: Patches, fill_value: &Scalar) -> VortexResult { + let fill_dtype = fill_value.dtype(); + let values_dtype = patches.values().dtype(); + + vortex_ensure!( + values_dtype.eq_ignore_nullability(fill_dtype), + "fill value, {:?}, should be instance of values dtype, {} but was {}.", + fill_value, + values_dtype, + fill_dtype, + ); + + if values_dtype == fill_dtype { + Ok(patches) + } else { + patches.cast_values(fill_dtype) + } + } + pub fn validate( patches: &Patches, fill_value: &Scalar, @@ -343,6 +381,8 @@ impl SparseData { } } + let values = Self::normalize_values_dtype(values, &fill_value)?; + // TODO(0ax1): handle chunk offsets let patches = Patches::new(len, 0, indices, values, None)?; Self::try_new_from_patches(patches, fill_value) @@ -350,13 +390,7 @@ impl SparseData { /// Build a new SparseArray from an existing set of patches. pub fn try_new_from_patches(patches: Patches, fill_value: Scalar) -> VortexResult { - vortex_ensure!( - fill_value.dtype() == patches.values().dtype(), - "fill value, {:?}, should be instance of values dtype, {} but was {}.", - fill_value, - patches.values().dtype(), - fill_value.dtype(), - ); + let patches = Self::normalize_patches_dtype(patches, &fill_value)?; let slots = Self::make_slots(&patches); @@ -429,10 +463,10 @@ impl SparseData { /// Optionally provided fill value will be respected if the array is less than 90% null. pub fn encode(array: &ArrayRef, fill_value: Option) -> VortexResult { if let Some(fill_value) = fill_value.as_ref() - && array.dtype() != fill_value.dtype() + && !array.dtype().eq_ignore_nullability(fill_value.dtype()) { vortex_bail!( - "Array and fill value types must match. got {} and {}", + "Array and fill value types must have the same base type. got {} and {}", array.dtype(), fill_value.dtype() ) @@ -477,7 +511,7 @@ impl SparseData { } let fill = if let Some(fill) = fill_value { - fill + fill.cast(array.dtype())? } else { // TODO(robert): Support other dtypes, only thing missing is getting most common value out of the array let (top_pvalue, _) = array From 0a30865d6c538328620b5e16be57b7cf0dfcc2cf Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Fri, 3 Apr 2026 14:02:28 -0400 Subject: [PATCH 10/15] Remove triggers Signed-off-by: Nicholas Gates --- encodings/fastlanes/goldenfiles/for.metadata | 1 + 1 file changed, 1 insertion(+) create mode 100644 encodings/fastlanes/goldenfiles/for.metadata diff --git a/encodings/fastlanes/goldenfiles/for.metadata b/encodings/fastlanes/goldenfiles/for.metadata new file mode 100644 index 00000000000..0e9da711b3e --- /dev/null +++ b/encodings/fastlanes/goldenfiles/for.metadata @@ -0,0 +1 @@ +þÿÿÿÿÿÿÿÿ \ No newline at end of file From 1dc30b2b45861cad460136f3ca92109e85e18fd4 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Fri, 3 Apr 2026 14:04:49 -0400 Subject: [PATCH 11/15] Remove triggers Signed-off-by: Nicholas Gates --- encodings/fastlanes/src/for/array/for_compress.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/encodings/fastlanes/src/for/array/for_compress.rs b/encodings/fastlanes/src/for/array/for_compress.rs index a1310083a4d..86bb6de2947 100644 --- a/encodings/fastlanes/src/for/array/for_compress.rs +++ b/encodings/fastlanes/src/for/array/for_compress.rs @@ -25,7 +25,7 @@ impl FoRData { let encoded = match_each_integer_ptype!(array.ptype(), |T| { compress_primitive::(array, T::try_from(&min)?)?.into_array() }); - Ok(FoR::try_new(encoded, min)?) + FoR::try_new(encoded, min) } } From 28a23ff50e41f7028c60d04b9834c82961fa1e6f Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Fri, 3 Apr 2026 14:18:03 -0400 Subject: [PATCH 12/15] Remove triggers Signed-off-by: Nicholas Gates --- vortex-array/src/arrays/struct_/array.rs | 65 +++++++++++------------- vortex-array/src/scalar_fn/fns/select.rs | 9 +--- vortex-web/crate/src/wasm.rs | 6 --- 3 files changed, 31 insertions(+), 49 deletions(-) diff --git a/vortex-array/src/arrays/struct_/array.rs b/vortex-array/src/arrays/struct_/array.rs index 7686bce9989..3f75a48e958 100644 --- a/vortex-array/src/arrays/struct_/array.rs +++ b/vortex-array/src/arrays/struct_/array.rs @@ -472,41 +472,6 @@ impl StructData { Self::try_from_iter_with_validity(iter, Validity::NonNullable) } - // TODO(aduffy): Add equivalent function to support field masks for nested column access. - /// Return a new StructArray with the given projection applied. - /// - /// Projection does not copy data arrays. Projection is defined by an ordinal array slice - /// which specifies the new ordering of columns in the struct. The projection can be used to - /// perform column re-ordering, deletion, or duplication at a logical level, without any data - /// copying. - pub fn project(&self, projection: &[FieldName]) -> VortexResult { - let mut children = Vec::with_capacity(projection.len()); - let mut names = Vec::with_capacity(projection.len()); - - for f_name in projection.iter() { - let idx = self - .names() - .iter() - .position(|name| name == f_name) - .ok_or_else(|| vortex_err!("Unknown field {f_name}"))?; - - names.push(self.names()[idx].clone()); - children.push( - self.slots[FIELDS_OFFSET + idx] - .as_ref() - .vortex_expect("StructArray field slot") - .clone(), - ); - } - - StructData::try_new( - FieldNames::from(names.as_slice()), - children, - self.len(), - self.validity(), - ) - } - /// Removes and returns a column from the struct array by name. /// If the column does not exist, returns `None`. pub fn remove_column(&mut self, name: impl Into) -> Option { @@ -638,6 +603,36 @@ impl Array { Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) } + // TODO(aduffy): Add equivalent function to support field masks for nested column access. + /// Return a new StructArray with the given projection applied. + /// + /// Projection does not copy data arrays. Projection is defined by an ordinal array slice + /// which specifies the new ordering of columns in the struct. The projection can be used to + /// perform column re-ordering, deletion, or duplication at a logical level, without any data + /// copying. + pub fn project(&self, projection: &[FieldName]) -> VortexResult { + let mut children = Vec::with_capacity(projection.len()); + let mut names = Vec::with_capacity(projection.len()); + + for f_name in projection { + let idx = self + .names() + .iter() + .position(|name| name == f_name) + .ok_or_else(|| vortex_err!("Unknown field {f_name}"))?; + + names.push(self.names()[idx].clone()); + children.push(self.unmasked_field(idx).clone()); + } + + Self::try_new( + FieldNames::from(names.as_slice()), + children, + self.len(), + self.validity(), + ) + } + /// Create a fieldless `StructArray` with the given length. pub fn new_fieldless_with_len(len: usize) -> Self { let data = StructData::new_fieldless_with_len(len); diff --git a/vortex-array/src/scalar_fn/fns/select.rs b/vortex-array/src/scalar_fn/fns/select.rs index 3c5a7cb8caf..7b323c4311c 100644 --- a/vortex-array/src/scalar_fn/fns/select.rs +++ b/vortex-array/src/scalar_fn/fns/select.rs @@ -18,9 +18,6 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; -use crate::array::Array; -use crate::array::ArrayParts; -use crate::arrays::Struct; use crate::arrays::StructArray; use crate::dtype::DType; use crate::dtype::FieldName; @@ -163,11 +160,7 @@ impl ScalarFnVTable for Select { } }?; - let dtype = result.dtype(); - let len = result.len(); - Array::try_from_parts(ArrayParts::new(Struct, dtype, len, result))? - .into_array() - .execute(ctx) + result.into_array().execute(ctx) } fn simplify( diff --git a/vortex-web/crate/src/wasm.rs b/vortex-web/crate/src/wasm.rs index 86459f34fa4..1139907f47f 100644 --- a/vortex-web/crate/src/wasm.rs +++ b/vortex-web/crate/src/wasm.rs @@ -384,8 +384,6 @@ impl VortexFileHandle { array_path: Vec, buffer_index: usize, ) -> Result { - use vortex::array::ArrayVisitor; - let ctx = self .array_read_ctx .as_ref() @@ -453,8 +451,6 @@ impl VortexFileHandle { array_path: Vec, row_limit: u32, ) -> Result { - use vortex::array::ArrayVisitor; - let ctx = self .array_read_ctx .as_ref() @@ -641,8 +637,6 @@ fn build_array_encoding_tree(parts: &SerializedArray, ctx: &ReadContext) -> Arra /// Recursively build the array encoding tree from a fully decoded array, /// extracting dtype, child names, and buffer names from the encoding vtables. fn build_array_encoding_tree_from_array(array: &ArrayRef) -> ArrayEncodingNodeJson { - use vortex::array::ArrayVisitor; - let encoding = array.encoding_id().to_string(); let dtype = array.dtype().to_string(); let buffer_names = array.buffer_names(); From 141d24181ed84e01abcea170add747c3dd367a01 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Fri, 3 Apr 2026 14:55:01 -0400 Subject: [PATCH 13/15] Remove triggers Signed-off-by: Nicholas Gates --- vortex-btrblocks/src/schemes/integer.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/vortex-btrblocks/src/schemes/integer.rs b/vortex-btrblocks/src/schemes/integer.rs index 70d86412194..9309637c158 100644 --- a/vortex-btrblocks/src/schemes/integer.rs +++ b/vortex-btrblocks/src/schemes/integer.rs @@ -335,6 +335,7 @@ impl Scheme for BitPackingScheme { return Ok(stats.source().clone().into_array()); } let packed = bitpack_encode(stats.source(), bw, Some(&histogram))?; + let packed_stats = packed.statistics().to_owned(); let ptype = packed.dtype().as_ptype(); let len = packed.len(); let nullability = packed.dtype().nullability(); @@ -353,6 +354,7 @@ impl Scheme for BitPackingScheme { parts.len, parts.offset, )? + .with_stats_set(packed_stats) .into_array()) } } @@ -869,6 +871,9 @@ mod scheme_selection_tests { use vortex_array::arrays::Constant; use vortex_array::arrays::Dict; use vortex_array::arrays::PrimitiveArray; + use vortex_array::expr::stats::Precision; + use vortex_array::expr::stats::Stat; + use vortex_array::expr::stats::StatsProviderExt; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexResult; @@ -907,6 +912,18 @@ mod scheme_selection_tests { let btr = BtrBlocksCompressor::default(); let compressed = btr.compress(&array.into_array())?; assert!(compressed.is::()); + assert_eq!( + compressed.statistics().get_as::(Stat::NullCount), + Some(Precision::exact(0u64)) + ); + assert_eq!( + compressed.statistics().get_as::(Stat::Min), + Some(Precision::exact(0u32)) + ); + assert_eq!( + compressed.statistics().get_as::(Stat::Max), + Some(Precision::exact(15u32)) + ); Ok(()) } From 1ff0c455983992350c238a83381a87301c3adfcb Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Fri, 3 Apr 2026 15:08:38 -0400 Subject: [PATCH 14/15] Remove triggers Signed-off-by: Nicholas Gates --- vortex-array/public-api.lock | 4 ++-- vortex-array/src/arrays/bool/vtable/mod.rs | 6 ++++++ vortex-array/src/display/extractors/stats.rs | 16 +++++++++------- vortex-array/src/display/mod.rs | 13 +++++++++++-- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 697653afd24..7e6487c34d1 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -3934,8 +3934,6 @@ pub fn vortex_array::arrays::struct_::StructData::new_fieldless_with_len(len: us pub unsafe fn vortex_array::arrays::struct_::StructData::new_unchecked(fields: impl core::convert::Into>, dtype: vortex_array::dtype::StructFields, length: usize, validity: vortex_array::validity::Validity) -> Self -pub fn vortex_array::arrays::struct_::StructData::project(&self, projection: &[vortex_array::dtype::FieldName]) -> vortex_error::VortexResult - pub fn vortex_array::arrays::struct_::StructData::remove_column(&mut self, name: impl core::convert::Into) -> core::option::Option pub fn vortex_array::arrays::struct_::StructData::struct_fields(&self) -> vortex_array::dtype::StructFields @@ -21654,6 +21652,8 @@ pub fn vortex_array::Array::new_fieldless_with_len pub unsafe fn vortex_array::Array::new_unchecked(fields: impl core::convert::Into>, dtype: vortex_array::dtype::StructFields, length: usize, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_array::Array::project(&self, projection: &[vortex_array::dtype::FieldName]) -> vortex_error::VortexResult + pub fn vortex_array::Array::try_from_iter, A: vortex_array::IntoArray, T: core::iter::traits::collect::IntoIterator>(iter: T) -> vortex_error::VortexResult pub fn vortex_array::Array::try_from_iter_with_validity, A: vortex_array::IntoArray, T: core::iter::traits::collect::IntoIterator>(iter: T, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult diff --git a/vortex-array/src/arrays/bool/vtable/mod.rs b/vortex-array/src/arrays/bool/vtable/mod.rs index bf9d13712e8..c2a5c25ca8c 100644 --- a/vortex-array/src/arrays/bool/vtable/mod.rs +++ b/vortex-array/src/arrays/bool/vtable/mod.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::fmt::Formatter; + use kernel::PARENT_KERNELS; use prost::Message; use vortex_error::VortexExpect; @@ -94,6 +96,10 @@ impl VTable for Bool { )) } + fn fmt_metadata(array: ArrayView<'_, Self>, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "BoolMetadata {{ offset: {} }}", array.offset) + } + fn validate(&self, data: &BoolData, dtype: &DType, len: usize) -> VortexResult<()> { vortex_ensure!( data.len() == len, diff --git a/vortex-array/src/display/extractors/stats.rs b/vortex-array/src/display/extractors/stats.rs index 70640e9619d..7f8fce7feb1 100644 --- a/vortex-array/src/display/extractors/stats.rs +++ b/vortex-array/src/display/extractors/stats.rs @@ -9,6 +9,7 @@ use crate::display::extractor::TreeContext; use crate::display::extractor::TreeExtractor; use crate::expr::stats::Stat; use crate::expr::stats::StatsProvider; +use crate::validity::Validity; /// Display wrapper for array statistics in compact format. /// @@ -39,16 +40,17 @@ impl fmt::Display for StatsDisplay<'_> { write!(f, "nulls={}", nc)?; } } else if self.0.dtype().is_nullable() { - match self.0.all_valid() { - Ok(true) => { + match self.0.validity() { + Ok(Validity::NonNullable | Validity::AllValid) => { sep(f)?; f.write_str("all_valid")?; } - Ok(false) => { - if self.0.all_invalid().unwrap_or(false) { - sep(f)?; - f.write_str("all_invalid")?; - } + Ok(Validity::AllInvalid) => { + sep(f)?; + f.write_str("all_invalid")?; + } + Ok(Validity::Array(_)) => { + // Avoid computing validity-array stats as a side effect of display. } Err(e) => { tracing::warn!("Failed to check validity: {e}"); diff --git a/vortex-array/src/display/mod.rs b/vortex-array/src/display/mod.rs index 79afebe0fbe..792135b1ded 100644 --- a/vortex-array/src/display/mod.rs +++ b/vortex-array/src/display/mod.rs @@ -649,6 +649,7 @@ mod test { use crate::IntoArray as _; use crate::arrays::BoolArray; use crate::arrays::ListArray; + use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; use crate::display::DISPLAY_LIMIT; use crate::dtype::FieldNames; @@ -665,7 +666,7 @@ mod test { let x = buffer![1, 2, 3, 4].into_array(); assert_eq!(x.display_values().to_string(), "[1i32, 2i32, 3i32, 4i32]"); - let x = crate::arrays::PrimitiveArray::from_iter( + let x = PrimitiveArray::from_iter( 0i32..i32::try_from(DISPLAY_LIMIT).unwrap() + 1, ) .into_array(); @@ -717,6 +718,14 @@ mod test { ); } + #[test] + fn test_display_tree_nullable_primitive_validity_child() { + let array = PrimitiveArray::from_option_iter([Some(1i64), Some(2), None, Some(3)]) + .into_array(); + let expected = "root: vortex.primitive(i64?, len=4) nbytes=33 B (100.00%)\n metadata: EmptyMetadata\n buffer: values host 32 B (align=8) (96.97%)\n validity: vortex.bool(bool, len=4) nbytes=1 B (3.03%)\n metadata: BoolMetadata { offset: 0 }\n buffer: bits host 1 B (align=1) (100.00%)\n"; + assert_eq!(format!("{}", array.display_tree()), expected); + } + #[test] fn test_table_display_primitive() { use crate::display::DisplayOptions; @@ -743,7 +752,7 @@ mod test { fn test_table_display() { use crate::display::DisplayOptions; - let array = crate::arrays::PrimitiveArray::from_option_iter(vec![ + let array = PrimitiveArray::from_option_iter(vec![ Some(-1), Some(-2), Some(-3), From 7c578bf1585504d9c8d07dd178278e96ebc6f196 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Fri, 3 Apr 2026 15:11:08 -0400 Subject: [PATCH 15/15] Remove triggers Signed-off-by: Nicholas Gates --- vortex-array/src/display/mod.rs | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/vortex-array/src/display/mod.rs b/vortex-array/src/display/mod.rs index 792135b1ded..42d9ace73d1 100644 --- a/vortex-array/src/display/mod.rs +++ b/vortex-array/src/display/mod.rs @@ -666,10 +666,8 @@ mod test { let x = buffer![1, 2, 3, 4].into_array(); assert_eq!(x.display_values().to_string(), "[1i32, 2i32, 3i32, 4i32]"); - let x = PrimitiveArray::from_iter( - 0i32..i32::try_from(DISPLAY_LIMIT).unwrap() + 1, - ) - .into_array(); + let x = + PrimitiveArray::from_iter(0i32..i32::try_from(DISPLAY_LIMIT).unwrap() + 1).into_array(); assert_eq!( x.display_values().to_string(), "[0i32, 1i32, 2i32, 3i32, 4i32, 5i32, 6i32, 7i32, 8i32, 9i32, 10i32, 11i32, 12i32, ..., 14i32, 15i32, 16i32]" @@ -720,8 +718,8 @@ mod test { #[test] fn test_display_tree_nullable_primitive_validity_child() { - let array = PrimitiveArray::from_option_iter([Some(1i64), Some(2), None, Some(3)]) - .into_array(); + let array = + PrimitiveArray::from_option_iter([Some(1i64), Some(2), None, Some(3)]).into_array(); let expected = "root: vortex.primitive(i64?, len=4) nbytes=33 B (100.00%)\n metadata: EmptyMetadata\n buffer: values host 32 B (align=8) (96.97%)\n validity: vortex.bool(bool, len=4) nbytes=1 B (3.03%)\n metadata: BoolMetadata { offset: 0 }\n buffer: bits host 1 B (align=1) (100.00%)\n"; assert_eq!(format!("{}", array.display_tree()), expected); } @@ -752,13 +750,8 @@ mod test { fn test_table_display() { use crate::display::DisplayOptions; - let array = PrimitiveArray::from_option_iter(vec![ - Some(-1), - Some(-2), - Some(-3), - None, - ]) - .into_array(); + let array = + PrimitiveArray::from_option_iter(vec![Some(-1), Some(-2), Some(-3), None]).into_array(); let struct_ = StructArray::try_from_iter_with_validity( [("x", buffer![1, 2, 3, 4].into_array()), ("y", array)],