diff --git a/docs/api/python/arrays.rst b/docs/api/python/arrays.rst index da6c6a7a449..55d4bc6d0b3 100644 --- a/docs/api/python/arrays.rst +++ b/docs/api/python/arrays.rst @@ -144,7 +144,7 @@ Registry and Serde .. autoclass:: vortex.ArrayContext :members: -.. autoclass:: vortex.ArrayParts +.. autoclass:: vortex.SerializedArray :members: diff --git a/encodings/alp/public-api.lock b/encodings/alp/public-api.lock index 9f380eb7450..6c0385166fb 100644 --- a/encodings/alp/public-api.lock +++ b/encodings/alp/public-api.lock @@ -12,7 +12,7 @@ impl vortex_alp::ALP pub fn vortex_alp::ALP::new(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option) -> vortex_alp::ALPArray -pub unsafe fn vortex_alp::ALP::new_unchecked(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option, dtype: vortex_array::dtype::DType) -> vortex_alp::ALPArray +pub unsafe fn vortex_alp::ALP::new_unchecked(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option) -> vortex_alp::ALPArray pub fn vortex_alp::ALP::try_new(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option) -> vortex_error::VortexResult @@ -28,8 +28,6 @@ impl vortex_array::array::vtable::VTable for vortex_alp::ALP pub type vortex_alp::ALP::ArrayData = vortex_alp::ALPData -pub type vortex_alp::ALP::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_alp::ALP::OperationsVTable = vortex_alp::ALP pub type vortex_alp::ALP::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -42,11 +40,7 @@ pub fn vortex_alp::ALP::buffer(_array: vortex_array::array::view::ArrayView<'_, pub fn vortex_alp::ALP::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_alp::ALP::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_alp::ALP::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_alp::ALP::dtype(array: &vortex_alp::ALPData) -> &vortex_array::dtype::DType +pub fn vortex_alp::ALP::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_alp::ALP::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -54,23 +48,17 @@ pub fn vortex_alp::ALP::execute_parent(array: vortex_array::array::view::ArrayVi pub fn vortex_alp::ALP::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_alp::ALP::len(array: &vortex_alp::ALPData) -> usize - -pub fn vortex_alp::ALP::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_alp::ALP::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_alp::ALP::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_alp::ALP::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_alp::ALP::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_alp::ALP::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_alp::ALP::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_alp::ALP::stats(array: &vortex_alp::ALPData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_alp::ALP::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_alp::ALP::validate(&self, data: &vortex_alp::ALPData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_alp::ALP::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -118,22 +106,14 @@ pub struct vortex_alp::ALPData impl vortex_alp::ALPData -pub fn vortex_alp::ALPData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_alp::ALPData::encoded(&self) -> &vortex_array::array::erased::ArrayRef pub fn vortex_alp::ALPData::exponents(&self) -> vortex_alp::Exponents -pub fn vortex_alp::ALPData::into_parts(self) -> (vortex_array::array::erased::ArrayRef, vortex_alp::Exponents, core::option::Option, vortex_array::dtype::DType) - -pub fn vortex_alp::ALPData::is_empty(&self) -> bool - -pub fn vortex_alp::ALPData::len(&self) -> usize +pub fn vortex_alp::ALPData::into_parts(self) -> (vortex_array::array::erased::ArrayRef, vortex_alp::Exponents, core::option::Option) pub fn vortex_alp::ALPData::patches(&self) -> core::option::Option -pub fn vortex_alp::ALPData::ptype(&self) -> vortex_array::dtype::ptype::PType - impl vortex_alp::ALPData pub fn vortex_alp::ALPData::new(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option) -> Self @@ -144,18 +124,10 @@ impl core::clone::Clone for vortex_alp::ALPData pub fn vortex_alp::ALPData::clone(&self) -> vortex_alp::ALPData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_alp::ALPData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_alp::ALPData pub fn vortex_alp::ALPData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_alp::ALPData - -pub fn vortex_alp::ALPData::into_array(self) -> vortex_array::array::erased::ArrayRef - pub struct vortex_alp::ALPMetadata impl core::clone::Clone for vortex_alp::ALPMetadata @@ -198,8 +170,6 @@ impl vortex_array::array::vtable::VTable for vortex_alp::ALPRD pub type vortex_alp::ALPRD::ArrayData = vortex_alp::ALPRDData -pub type vortex_alp::ALPRD::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_alp::ALPRD::OperationsVTable = vortex_alp::ALPRD pub type vortex_alp::ALPRD::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -212,11 +182,7 @@ pub fn vortex_alp::ALPRD::buffer(_array: vortex_array::array::view::ArrayView<'_ pub fn vortex_alp::ALPRD::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_alp::ALPRD::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_alp::ALPRD::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_alp::ALPRD::dtype(array: &vortex_alp::ALPRDData) -> &vortex_array::dtype::DType +pub fn vortex_alp::ALPRD::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_alp::ALPRD::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -224,23 +190,17 @@ pub fn vortex_alp::ALPRD::execute_parent(array: vortex_array::array::view::Array pub fn vortex_alp::ALPRD::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_alp::ALPRD::len(array: &vortex_alp::ALPRDData) -> usize - -pub fn vortex_alp::ALPRD::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_alp::ALPRD::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_alp::ALPRD::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_alp::ALPRD::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_alp::ALPRD::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_alp::ALPRD::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_alp::ALPRD::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_alp::ALPRD::stats(array: &vortex_alp::ALPRDData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_alp::ALPRD::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_alp::ALPRD::validate(&self, data: &vortex_alp::ALPRDData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_alp::ALPRD::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -272,37 +232,11 @@ impl vortex_array::scalar_fn::fns::mask::kernel::MaskReduce for vortex_alp::ALPR pub fn vortex_alp::ALPRD::mask(array: vortex_array::array::view::ArrayView<'_, Self>, mask: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_alp::ALPRDArrayParts - -pub vortex_alp::ALPRDArrayParts::dtype: vortex_array::dtype::DType - -pub vortex_alp::ALPRDArrayParts::left_parts: vortex_array::array::erased::ArrayRef - -pub vortex_alp::ALPRDArrayParts::left_parts_dictionary: vortex_buffer::buffer::Buffer - -pub vortex_alp::ALPRDArrayParts::left_parts_patches: core::option::Option - -pub vortex_alp::ALPRDArrayParts::right_parts: vortex_array::array::erased::ArrayRef - -impl core::clone::Clone for vortex_alp::ALPRDArrayParts - -pub fn vortex_alp::ALPRDArrayParts::clone(&self) -> vortex_alp::ALPRDArrayParts - -impl core::fmt::Debug for vortex_alp::ALPRDArrayParts - -pub fn vortex_alp::ALPRDArrayParts::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub struct vortex_alp::ALPRDData impl vortex_alp::ALPRDData -pub fn vortex_alp::ALPRDData::dtype(&self) -> &vortex_array::dtype::DType - -pub fn vortex_alp::ALPRDData::into_parts(self) -> vortex_alp::ALPRDArrayParts - -pub fn vortex_alp::ALPRDData::is_empty(&self) -> bool - -pub fn vortex_alp::ALPRDData::is_f32(&self) -> bool +pub fn vortex_alp::ALPRDData::into_parts(self) -> vortex_alp::ALPRDDataParts pub fn vortex_alp::ALPRDData::left_parts(&self) -> &vortex_array::array::erased::ArrayRef @@ -310,8 +244,6 @@ pub fn vortex_alp::ALPRDData::left_parts_dictionary(&self) -> &vortex_buffer::bu pub fn vortex_alp::ALPRDData::left_parts_patches(&self) -> core::option::Option -pub fn vortex_alp::ALPRDData::len(&self) -> usize - pub fn vortex_alp::ALPRDData::replace_left_parts_patches(&mut self, patches: core::option::Option) pub fn vortex_alp::ALPRDData::right_bit_width(&self) -> u8 @@ -324,17 +256,27 @@ impl core::clone::Clone for vortex_alp::ALPRDData pub fn vortex_alp::ALPRDData::clone(&self) -> vortex_alp::ALPRDData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_alp::ALPRDData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_alp::ALPRDData pub fn vortex_alp::ALPRDData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_alp::ALPRDData +pub struct vortex_alp::ALPRDDataParts + +pub vortex_alp::ALPRDDataParts::left_parts: vortex_array::array::erased::ArrayRef + +pub vortex_alp::ALPRDDataParts::left_parts_dictionary: vortex_buffer::buffer::Buffer + +pub vortex_alp::ALPRDDataParts::left_parts_patches: core::option::Option + +pub vortex_alp::ALPRDDataParts::right_parts: vortex_array::array::erased::ArrayRef + +impl core::clone::Clone for vortex_alp::ALPRDDataParts + +pub fn vortex_alp::ALPRDDataParts::clone(&self) -> vortex_alp::ALPRDDataParts + +impl core::fmt::Debug for vortex_alp::ALPRDDataParts -pub fn vortex_alp::ALPRDData::into_array(self) -> vortex_array::array::erased::ArrayRef +pub fn vortex_alp::ALPRDDataParts::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct vortex_alp::ALPRDMetadata diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index a5930c9b1ce..93dfb8d3932 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -4,19 +4,18 @@ use std::fmt::Debug; use std::hash::Hash; +use prost::Message; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; -use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; -use vortex_array::SerializeMetadata; use vortex_array::arrays::Primitive; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; @@ -26,7 +25,6 @@ use vortex_array::patches::PatchesMetadata; use vortex_array::require_child; use vortex_array::require_patches; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityChild; @@ -49,28 +47,15 @@ vtable!(ALP, ALP, ALPData); impl VTable for ALP { type ArrayData = ALPData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; - fn vtable(_array: &Self::ArrayData) -> &Self { - &ALP - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ALPData) -> usize { - array.encoded().len() - } - - fn dtype(array: &ALPData) -> &DType { - &array.dtype - } - - fn stats(array: &ALPData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &ALPData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate_against_outer(dtype, len) } fn array_hash(array: &ALPData, state: &mut H, precision: Precision) { @@ -97,41 +82,31 @@ impl VTable for ALP { None } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { let exponents = array.exponents(); - Ok(ProstMetadata(ALPMetadata { - exp_e: exponents.e as u32, - exp_f: exponents.f as u32, - patches: array - .patches() - .map(|p| p.to_metadata(array.len(), array.dtype())) - .transpose()?, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) - } - - fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata( - as DeserializeMetadata>::deserialize(bytes)?, + Ok(Some( + ALPMetadata { + exp_e: exponents.e as u32, + exp_f: exponents.f as u32, + patches: array + .patches() + .map(|p| p.to_metadata(array.len(), array.dtype())) + .transpose()?, + } + .encode_to_vec(), )) } - fn build( + fn deserialize( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = ALPMetadata::decode(metadata)?; let encoded_ptype = match &dtype { DType::Primitive(PType::F32, n) => DType::Primitive(PType::I32, *n), DType::Primitive(PType::F64, n) => DType::Primitive(PType::I64, *n), @@ -243,9 +218,7 @@ pub struct ALPData { pub(super) slots: Vec>, patch_offset: Option, patch_offset_within_chunk: Option, - dtype: DType, exponents: Exponents, - stats_set: ArrayStats, } #[derive(Clone, Debug)] @@ -266,7 +239,7 @@ pub struct ALPMetadata { } impl ALPData { - fn validate( + fn validate_components( encoded: &ArrayRef, exponents: Exponents, patches: Option<&Patches>, @@ -317,6 +290,38 @@ impl ALPData { Ok(()) } + fn logical_dtype(encoded: &ArrayRef) -> VortexResult { + match encoded.dtype() { + DType::Primitive(PType::I32, nullability) => { + Ok(DType::Primitive(PType::F32, *nullability)) + } + DType::Primitive(PType::I64, nullability) => { + Ok(DType::Primitive(PType::F64, *nullability)) + } + _ => vortex_bail!("ALP encoded ints have invalid DType {}", encoded.dtype(),), + } + } + + fn validate_against_outer(&self, dtype: &DType, len: usize) -> VortexResult<()> { + let patches = self.patches(); + let logical_dtype = Self::logical_dtype(self.encoded())?; + Self::validate_components(self.encoded(), self.exponents, patches.as_ref())?; + + vortex_ensure!( + self.encoded().len() == len, + "ALP encoded len {} != outer len {len}", + self.encoded().len(), + ); + vortex_ensure!( + &logical_dtype == dtype, + "ALP dtype {} does not match encoded logical dtype {}", + dtype, + logical_dtype, + ); + + Ok(()) + } + /// Validate that any patches provided are valid for the ALPArray. fn validate_patches(patches: &Patches, encoded: &ArrayRef) -> VortexResult<()> { vortex_ensure!( @@ -368,7 +373,7 @@ impl ALPData { /// # Examples /// /// ``` - /// # use vortex_alp::{ALPData, ALPArray, Exponents}; + /// # use vortex_alp::{ALP, ALPData, Exponents}; /// # use vortex_array::IntoArray; /// # use vortex_buffer::buffer; /// @@ -389,11 +394,11 @@ impl ALPData { /// assert!(result.is_err()); /// /// // Success! - /// let value = ALPArray::try_from_data(ALPData::try_new( + /// let value = ALP::try_new( /// buffer![0i32].into_array(), /// Exponents { e: 1, f: 1 }, /// None - /// ).unwrap()).unwrap(); + /// ).unwrap(); /// /// assert_eq!(value.scalar_at(0).unwrap(), 0f32.into()); /// ``` @@ -402,13 +407,7 @@ impl ALPData { exponents: Exponents, patches: Option, ) -> VortexResult { - Self::validate(&encoded, exponents, patches.as_ref())?; - - let dtype = match encoded.dtype() { - DType::Primitive(PType::I32, nullability) => DType::Primitive(PType::F32, *nullability), - DType::Primitive(PType::I64, nullability) => DType::Primitive(PType::F64, *nullability), - _ => unreachable!(), - }; + Self::validate_components(&encoded, exponents, patches.as_ref())?; let slots = Self::make_slots(&encoded, &patches); let (patch_offset, patch_offset_within_chunk) = match &patches { @@ -417,12 +416,10 @@ impl ALPData { }; Ok(Self { - dtype, slots, - exponents, patch_offset, patch_offset_within_chunk, - stats_set: Default::default(), + exponents, }) } @@ -434,7 +431,6 @@ impl ALPData { encoded: ArrayRef, exponents: Exponents, patches: Option, - dtype: DType, ) -> Self { let slots = Self::make_slots(&encoded, &patches); let (patch_offset, patch_offset_within_chunk) = match &patches { @@ -443,12 +439,10 @@ impl ALPData { }; Self { - dtype, slots, - exponents, patch_offset, patch_offset_within_chunk, - stats_set: Default::default(), + exponents, } } } @@ -456,8 +450,16 @@ impl ALPData { /// Constructors for [`ALPArray`]. impl ALP { pub fn new(encoded: ArrayRef, exponents: Exponents, patches: Option) -> ALPArray { - Array::try_from_data(ALPData::new(encoded, exponents, patches)) - .vortex_expect("ALPData is always valid") + let dtype = ALPData::logical_dtype(&encoded).vortex_expect("ALP encoded dtype"); + let len = encoded.len(); + unsafe { + Array::from_parts_unchecked(ArrayParts::new( + ALP, + dtype, + len, + ALPData::new(encoded, exponents, patches), + )) + } } pub fn try_new( @@ -465,7 +467,10 @@ impl ALP { exponents: Exponents, patches: Option, ) -> VortexResult { - Array::try_from_data(ALPData::try_new(encoded, exponents, patches)?) + let dtype = ALPData::logical_dtype(&encoded)?; + let len = encoded.len(); + let data = ALPData::try_new(encoded, exponents, patches)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(ALP, dtype, len, data)) }) } /// # Safety @@ -474,29 +479,15 @@ impl ALP { encoded: ArrayRef, exponents: Exponents, patches: Option, - dtype: DType, ) -> ALPArray { - Array::try_from_data(unsafe { ALPData::new_unchecked(encoded, exponents, patches, dtype) }) - .vortex_expect("ALPData is always valid") + let dtype = ALPData::logical_dtype(&encoded).vortex_expect("ALP encoded dtype"); + let len = encoded.len(); + let data = unsafe { ALPData::new_unchecked(encoded, exponents, patches) }; + unsafe { Array::from_parts_unchecked(ArrayParts::new(ALP, dtype, len, data)) } } } impl ALPData { - /// Returns the number of elements in the array. - pub fn len(&self) -> usize { - self.encoded().len() - } - - /// Returns `true` if the array contains no elements. - pub fn is_empty(&self) -> bool { - self.encoded().len() == 0 - } - - /// Returns the logical data type of the array. - pub fn dtype(&self) -> &DType { - &self.dtype - } - fn make_slots(encoded: &ArrayRef, patches: &Option) -> Vec> { let (patch_indices, patch_values, patch_chunk_offsets) = match patches { Some(p) => ( @@ -514,10 +505,6 @@ impl ALPData { ] } - pub fn ptype(&self) -> PType { - self.dtype.as_ptype() - } - pub fn encoded(&self) -> &ArrayRef { self.slots[ENCODED_SLOT] .as_ref() @@ -555,12 +542,12 @@ impl ALPData { /// Consumes the array and returns its parts. #[inline] - pub fn into_parts(mut self) -> (ArrayRef, Exponents, Option, DType) { + pub fn into_parts(mut self) -> (ArrayRef, Exponents, Option) { let patches = self.patches(); let encoded = self.slots[ENCODED_SLOT] .take() .vortex_expect("ALPArray encoded slot"); - (encoded, self.exponents, patches, self.dtype) + (encoded, self.exponents, patches) } } @@ -884,12 +871,11 @@ mod tests { .unwrap(); // Build a new ALPArray with the same encoded data but patches without chunk_offsets. - let alp_without_chunk_offsets = ALPArray::try_from_data(ALPData::new( + let alp_without_chunk_offsets = ALP::new( normally_encoded.encoded().clone(), normally_encoded.exponents(), Some(patches_without_chunk_offsets), - )) - .vortex_expect("ALPData is always valid"); + ); // The legacy decompress_into_array path should work correctly. let result_legacy = decompress_into_array( diff --git a/encodings/alp/src/alp/compress.rs b/encodings/alp/src/alp/compress.rs index e8992c7a2a4..a55baf02698 100644 --- a/encodings/alp/src/alp/compress.rs +++ b/encodings/alp/src/alp/compress.rs @@ -47,14 +47,7 @@ pub fn alp_encode(parray: &PrimitiveArray, exponents: Option) -> Vort }; // SAFETY: alp_encode_components_typed must return well-formed components - unsafe { - Ok(ALP::new_unchecked( - encoded, - exponents, - patches, - parray.dtype().clone(), - )) - } + unsafe { Ok(ALP::new_unchecked(encoded, exponents, patches)) } } #[expect( diff --git a/encodings/alp/src/alp/compute/between.rs b/encodings/alp/src/alp/compute/between.rs index ff0b4da8943..efaaf747768 100644 --- a/encodings/alp/src/alp/compute/between.rs +++ b/encodings/alp/src/alp/compute/between.rs @@ -38,7 +38,7 @@ impl BetweenReduce for ALP { let nullability = array.dtype().nullability() | lower.dtype().nullability() | upper.dtype().nullability(); - match_each_alp_float_ptype!(array.ptype(), |F| { + match_each_alp_float_ptype!(array.dtype().as_ptype(), |F| { between_impl::( array, F::try_from(&lower)?, diff --git a/encodings/alp/src/alp/compute/cast.rs b/encodings/alp/src/alp/compute/cast.rs index ebca5e3fe0f..bd8a8b38381 100644 --- a/encodings/alp/src/alp/compute/cast.rs +++ b/encodings/alp/src/alp/compute/cast.rs @@ -45,8 +45,7 @@ impl CastReduce for ALP { // SAFETY: casting nullability doesn't alter the invariants unsafe { Ok(Some( - ALP::new_unchecked(new_encoded, array.exponents(), new_patches, dtype.clone()) - .into_array(), + ALP::new_unchecked(new_encoded, array.exponents(), new_patches).into_array(), )) } } else { diff --git a/encodings/alp/src/alp/compute/filter.rs b/encodings/alp/src/alp/compute/filter.rs index a4e3c4371d9..3daa6d53d7b 100644 --- a/encodings/alp/src/alp/compute/filter.rs +++ b/encodings/alp/src/alp/compute/filter.rs @@ -30,7 +30,6 @@ impl FilterKernel for ALP { array.encoded().filter(mask.clone())?, array.exponents(), patches, - array.dtype().clone(), ) .into_array(), )) diff --git a/encodings/alp/src/alp/decompress.rs b/encodings/alp/src/alp/decompress.rs index 0e7c1a5e7a0..e118b9705cd 100644 --- a/encodings/alp/src/alp/decompress.rs +++ b/encodings/alp/src/alp/decompress.rs @@ -27,7 +27,8 @@ pub fn decompress_into_array( array: ALPArray, ctx: &mut ExecutionCtx, ) -> VortexResult { - let (encoded, exponents, patches, dtype) = array.into_data().into_parts(); + let dtype = array.dtype().clone(); + let (encoded, exponents, patches) = array.into_data().into_parts(); if let Some(ref patches) = patches && let Some(chunk_offsets) = patches.chunk_offsets() { @@ -59,7 +60,8 @@ pub fn decompress_into_array( /// /// A `PrimitiveArray` containing the decompressed floating-point values with all patches applied. pub fn execute_decompress(array: ALPArray, ctx: &mut ExecutionCtx) -> VortexResult { - let (encoded, exponents, patches, dtype) = array.into_data().into_parts(); + let dtype = array.dtype().clone(); + let (encoded, exponents, patches) = array.into_data().into_parts(); if let Some(ref patches) = patches && let Some(chunk_offsets) = patches.chunk_offsets() { diff --git a/encodings/alp/src/alp/mod.rs b/encodings/alp/src/alp/mod.rs index b68d58b2ad3..7d5049b59e3 100644 --- a/encodings/alp/src/alp/mod.rs +++ b/encodings/alp/src/alp/mod.rs @@ -22,7 +22,7 @@ mod rules; #[cfg(test)] mod tests { - use vortex_array::ProstMetadata; + use prost::Message; use vortex_array::dtype::PType; use vortex_array::patches::PatchesMetadata; use vortex_array::test_harness::check_metadata; @@ -34,7 +34,7 @@ mod tests { fn test_alp_metadata() { check_metadata( "alp.metadata", - ProstMetadata(ALPMetadata { + &ALPMetadata { patches: Some(PatchesMetadata::new( usize::MAX, usize::MAX, @@ -45,7 +45,8 @@ mod tests { )), exp_e: u32::MAX, exp_f: u32::MAX, - }), + } + .encode_to_vec(), ); } } diff --git a/encodings/alp/src/alp/ops.rs b/encodings/alp/src/alp/ops.rs index 9538e3c60a1..25cf19a3e55 100644 --- a/encodings/alp/src/alp/ops.rs +++ b/encodings/alp/src/alp/ops.rs @@ -26,7 +26,7 @@ impl OperationsVTable for ALP { let encoded_val = array.encoded().scalar_at(index)?; - Ok(match_each_alp_float_ptype!(array.ptype(), |T| { + Ok(match_each_alp_float_ptype!(array.dtype().as_ptype(), |T| { let encoded_val: ::ALPInt = (&encoded_val).try_into().vortex_expect("invalid ALPInt"); Scalar::primitive( diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index eca56a97a37..b16470372bc 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -5,19 +5,18 @@ use std::fmt::Debug; use std::hash::Hash; use itertools::Itertools; +use prost::Message; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; -use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; -use vortex_array::SerializeMetadata; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; @@ -29,7 +28,6 @@ use vortex_array::patches::PatchesMetadata; use vortex_array::require_child; use vortex_array::require_patches; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; @@ -67,28 +65,15 @@ pub struct ALPRDMetadata { impl VTable for ALPRD { type ArrayData = ALPRDData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; - fn vtable(_array: &Self::ArrayData) -> &Self { - &ALPRD - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ALPRDData) -> usize { - array.left_parts().len() - } - - fn dtype(array: &ALPRDData) -> &DType { - &array.dtype - } - - fn stats(array: &ALPRDData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &ALPRDData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate_against_outer(dtype, len) } fn array_hash(array: &ALPRDData, state: &mut H, precision: Precision) { @@ -123,48 +108,38 @@ impl VTable for ALPRD { None } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { let dict = array .left_parts_dictionary() .iter() .map(|&i| i as u32) .collect::>(); - Ok(ProstMetadata(ALPRDMetadata { - right_bit_width: array.right_bit_width() as u32, - dict_len: array.left_parts_dictionary().len() as u32, - dict, - left_parts_ptype: array.left_parts().dtype().as_ptype() as i32, - patches: array - .left_parts_patches() - .map(|p| p.to_metadata(array.len(), array.left_parts().dtype())) - .transpose()?, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) - } - - fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata( - as DeserializeMetadata>::deserialize(bytes)?, + Ok(Some( + ALPRDMetadata { + right_bit_width: array.right_bit_width() as u32, + dict_len: array.left_parts_dictionary().len() as u32, + dict, + left_parts_ptype: array.left_parts().dtype().as_ptype() as i32, + patches: array + .left_parts_patches() + .map(|p| p.to_metadata(array.len(), array.left_parts().dtype())) + .transpose()?, + } + .encode_to_vec(), )) } - fn build( + fn deserialize( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = ALPRDMetadata::decode(metadata)?; if children.len() < 2 { vortex_bail!( "Expected at least 2 children for ALPRD encoding, found {}", @@ -172,10 +147,10 @@ impl VTable for ALPRD { ); } - let left_parts_dtype = DType::Primitive(metadata.0.left_parts_ptype(), dtype.nullability()); + let left_parts_dtype = DType::Primitive(metadata.left_parts_ptype(), dtype.nullability()); let left_parts = children.get(0, &left_parts_dtype, len)?; - let left_parts_dictionary: Buffer = metadata.0.dict.as_slice() - [0..metadata.0.dict_len as usize] + let left_parts_dictionary: Buffer = metadata.dict.as_slice() + [0..metadata.dict_len as usize] .iter() .map(|&i| { u16::try_from(i) @@ -195,7 +170,6 @@ impl VTable for ALPRD { let right_parts = children.get(1, &right_parts_dtype, len)?; let left_parts_patches = metadata - .0 .patches .map(|p| { let indices = children.get(2, &p.indices_dtype()?, p.len()?)?; @@ -217,10 +191,10 @@ impl VTable for ALPRD { left_parts, left_parts_dictionary, right_parts, - u8::try_from(metadata.0.right_bit_width).map_err(|_| { + u8::try_from(metadata.right_bit_width).map_err(|_| { vortex_err!( "right_bit_width {} out of u8 range", - metadata.0.right_bit_width + metadata.right_bit_width ) })?, left_parts_patches, @@ -276,14 +250,13 @@ impl VTable for ALPRD { LP_PATCH_CHUNK_OFFSETS_SLOT ); + let dtype = array.dtype().clone(); let right_bit_width = array.right_bit_width(); - let ALPRDArrayParts { + let ALPRDDataParts { left_parts, right_parts, left_parts_dictionary, left_parts_patches, - dtype, - .. } = array.into_data().into_parts(); let ptype = dtype.as_ptype(); @@ -368,17 +341,14 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = [ #[derive(Clone, Debug)] pub struct ALPRDData { - dtype: DType, slots: Vec>, left_parts_patches: Option, left_parts_dictionary: Buffer, right_bit_width: u8, - stats_set: ArrayStats, } #[derive(Clone, Debug)] -pub struct ALPRDArrayParts { - pub dtype: DType, +pub struct ALPRDDataParts { pub left_parts: ArrayRef, pub left_parts_patches: Option, pub left_parts_dictionary: Buffer, @@ -399,14 +369,21 @@ impl ALPRD { right_bit_width: u8, left_parts_patches: Option, ) -> VortexResult { - Array::try_from_data(ALPRDData::try_new( + let len = left_parts.len(); + let logical_dtype = dtype.clone(); + let data = ALPRDData::try_new( dtype, left_parts, left_parts_dictionary, right_parts, right_bit_width, left_parts_patches, - )?) + )?; + Ok( + unsafe { + Array::from_parts_unchecked(ArrayParts::new(ALPRD, logical_dtype, len, data)) + }, + ) } /// # Safety @@ -419,7 +396,9 @@ impl ALPRD { right_bit_width: u8, left_parts_patches: Option, ) -> ALPRDArray { - Array::try_from_data(unsafe { + let len = left_parts.len(); + let logical_dtype = dtype.clone(); + let data = unsafe { ALPRDData::new_unchecked( dtype, left_parts, @@ -428,33 +407,33 @@ impl ALPRD { right_bit_width, left_parts_patches, ) - }) - .vortex_expect("ALPRDData is always valid") + }; + unsafe { Array::from_parts_unchecked(ArrayParts::new(ALPRD, logical_dtype, len, data)) } } } impl ALPRDData { - /// Build a new `ALPRDArray` from components. - pub fn try_new( - dtype: DType, - left_parts: ArrayRef, - left_parts_dictionary: Buffer, - right_parts: ArrayRef, - right_bit_width: u8, - left_parts_patches: Option, - ) -> VortexResult { + fn validate_parts( + dtype: &DType, + len: usize, + left_parts: &ArrayRef, + right_parts: &ArrayRef, + left_parts_patches: Option<&Patches>, + ) -> VortexResult<()> { if !dtype.is_float() { vortex_bail!("ALPRDArray given invalid DType ({dtype})"); } - let len = left_parts.len(); - if right_parts.len() != len { - vortex_bail!( - "left_parts (len {}) and right_parts (len {}) must be of same length", - len, - right_parts.len() - ); - } + vortex_ensure!( + left_parts.len() == len, + "left_parts len {} != outer len {len}", + left_parts.len(), + ); + vortex_ensure!( + right_parts.len() == len, + "right_parts len {} != outer len {len}", + right_parts.len(), + ); if !left_parts.dtype().is_unsigned_int() { vortex_bail!("left_parts dtype must be uint"); @@ -468,11 +447,64 @@ impl ALPRDData { ); } - // we enforce right_parts to be non-nullable uint - if !right_parts.dtype().is_unsigned_int() || right_parts.dtype().is_nullable() { - vortex_bail!(MismatchedTypes: "non-nullable uint", right_parts.dtype()); + let expected_right_parts_dtype = match dtype { + DType::Primitive(PType::F32, _) => { + DType::Primitive(PType::U32, Nullability::NonNullable) + } + DType::Primitive(PType::F64, _) => { + DType::Primitive(PType::U64, Nullability::NonNullable) + } + _ => vortex_bail!("Expected f32 or f64 dtype, got {:?}", dtype), + }; + vortex_ensure!( + right_parts.dtype() == &expected_right_parts_dtype, + "right_parts dtype {} does not match expected {}", + right_parts.dtype(), + expected_right_parts_dtype, + ); + + if let Some(patches) = left_parts_patches { + vortex_ensure!( + patches.array_len() == len, + "patches array_len {} != outer len {len}", + patches.array_len(), + ); + vortex_ensure!( + patches.dtype() == left_parts.dtype(), + "patches dtype {} does not match left_parts dtype {}", + patches.dtype(), + left_parts.dtype(), + ); + vortex_ensure!( + patches.values().all_valid()?, + "patches must be all valid: {}", + patches.values() + ); } + Ok(()) + } + + fn validate_against_outer(&self, dtype: &DType, len: usize) -> VortexResult<()> { + Self::validate_parts( + dtype, + len, + self.left_parts(), + self.right_parts(), + self.left_parts_patches.as_ref(), + ) + } + + /// Build a new `ALPRDArray` from components. + pub fn try_new( + dtype: DType, + left_parts: ArrayRef, + left_parts_dictionary: Buffer, + right_parts: ArrayRef, + right_bit_width: u8, + left_parts_patches: Option, + ) -> VortexResult { + let len = left_parts.len(); let left_parts_patches = left_parts_patches .map(|patches| { if !patches.values().all_valid()? { @@ -488,22 +520,28 @@ impl ALPRDData { }) .transpose()?; + Self::validate_parts( + &dtype, + len, + &left_parts, + &right_parts, + left_parts_patches.as_ref(), + )?; + let slots = Self::make_slots(&left_parts, &right_parts, &left_parts_patches); Ok(Self { - dtype, slots, + left_parts_patches, left_parts_dictionary, right_bit_width, - left_parts_patches, - stats_set: Default::default(), }) } /// Build a new `ALPRDArray` from components. This does not perform any validation, and instead /// it constructs it from parts. pub(crate) unsafe fn new_unchecked( - dtype: DType, + _dtype: DType, left_parts: ArrayRef, left_parts_dictionary: Buffer, right_parts: ArrayRef, @@ -513,12 +551,10 @@ impl ALPRDData { let slots = Self::make_slots(&left_parts, &right_parts, &left_parts_patches); Self { - dtype, slots, left_parts_patches, left_parts_dictionary, right_bit_width, - stats_set: Default::default(), } } @@ -545,15 +581,14 @@ impl ALPRDData { } /// Return all the owned parts of the array - pub fn into_parts(mut self) -> ALPRDArrayParts { + pub fn into_parts(mut self) -> ALPRDDataParts { let left_parts = self.slots[LEFT_PARTS_SLOT] .take() .vortex_expect("ALPRDArray left_parts slot"); let right_parts = self.slots[RIGHT_PARTS_SLOT] .take() .vortex_expect("ALPRDArray right_parts slot"); - ALPRDArrayParts { - dtype: self.dtype, + ALPRDDataParts { left_parts, left_parts_patches: self.left_parts_patches, left_parts_dictionary: self.left_parts_dictionary, @@ -561,29 +596,6 @@ impl ALPRDData { } } - /// Returns the number of elements in the array. - pub fn len(&self) -> usize { - self.left_parts().len() - } - - /// Returns `true` if the array contains no elements. - pub fn is_empty(&self) -> bool { - self.left_parts().len() == 0 - } - - /// Returns the logical data type of the array. - pub fn dtype(&self) -> &DType { - &self.dtype - } - - /// Returns true if logical type of the array values is f32. - /// - /// Returns false if the logical type of the array values is f64. - #[inline] - pub fn is_f32(&self) -> bool { - matches!(&self.dtype, DType::Primitive(PType::F32, _)) - } - /// The leftmost (most significant) bits of the floating point values stored in the array. /// /// These are bit-packed and dictionary encoded, and cannot directly be interpreted without @@ -642,8 +654,8 @@ impl ValidityChild for ALPRD { #[cfg(test)] mod test { + use prost::Message; use rstest::rstest; - use vortex_array::ProstMetadata; use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -687,7 +699,7 @@ mod test { fn test_alprd_metadata() { check_metadata( "alprd.metadata", - ProstMetadata(ALPRDMetadata { + &ALPRDMetadata { right_bit_width: u32::MAX, patches: Some(PatchesMetadata::new( usize::MAX, @@ -700,7 +712,8 @@ mod test { dict: Vec::new(), left_parts_ptype: PType::U64 as i32, dict_len: 8, - }), + } + .encode_to_vec(), ); } } diff --git a/encodings/alp/src/alp_rd/ops.rs b/encodings/alp/src/alp_rd/ops.rs index 63dc3fc5d89..6f77fdbd843 100644 --- a/encodings/alp/src/alp_rd/ops.rs +++ b/encodings/alp/src/alp_rd/ops.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; +use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; use vortex_array::vtable::OperationsVTable; use vortex_error::VortexExpect; @@ -39,7 +40,7 @@ impl OperationsVTable for ALPRD { }; // combine left and right values - Ok(if array.is_f32() { + Ok(if array.dtype().as_ptype() == PType::F32 { let right: u32 = array .right_parts() .scalar_at(index)? diff --git a/encodings/bytebool/public-api.lock b/encodings/bytebool/public-api.lock index 5f05421a6d6..74eed2ce798 100644 --- a/encodings/bytebool/public-api.lock +++ b/encodings/bytebool/public-api.lock @@ -6,8 +6,12 @@ impl vortex_bytebool::ByteBool pub const vortex_bytebool::ByteBool::ID: vortex_array::array::ArrayId +pub fn vortex_bytebool::ByteBool::from_option_vec(data: alloc::vec::Vec>) -> vortex_bytebool::ByteBoolArray + pub fn vortex_bytebool::ByteBool::from_vec>(data: alloc::vec::Vec, validity: V) -> vortex_bytebool::ByteBoolArray +pub fn vortex_bytebool::ByteBool::new(buffer: vortex_array::buffer::BufferHandle, validity: vortex_array::validity::Validity) -> vortex_bytebool::ByteBoolArray + impl core::clone::Clone for vortex_bytebool::ByteBool pub fn vortex_bytebool::ByteBool::clone(&self) -> vortex_bytebool::ByteBool @@ -20,8 +24,6 @@ impl vortex_array::array::vtable::VTable for vortex_bytebool::ByteBool pub type vortex_bytebool::ByteBool::ArrayData = vortex_bytebool::ByteBoolData -pub type vortex_bytebool::ByteBool::Metadata = vortex_array::metadata::EmptyMetadata - pub type vortex_bytebool::ByteBool::OperationsVTable = vortex_bytebool::ByteBool pub type vortex_bytebool::ByteBool::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromValidityHelper @@ -34,11 +36,7 @@ pub fn vortex_bytebool::ByteBool::buffer(array: vortex_array::array::view::Array pub fn vortex_bytebool::ByteBool::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_bytebool::ByteBool::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_bytebool::ByteBool::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_bytebool::ByteBool::dtype(array: &vortex_bytebool::ByteBoolData) -> &vortex_array::dtype::DType +pub fn vortex_bytebool::ByteBool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_bytebool::ByteBool::execute(array: vortex_array::array::typed::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -46,23 +44,17 @@ pub fn vortex_bytebool::ByteBool::execute_parent(array: vortex_array::array::vie pub fn vortex_bytebool::ByteBool::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_bytebool::ByteBool::len(array: &vortex_bytebool::ByteBoolData) -> usize - -pub fn vortex_bytebool::ByteBool::metadata(_array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_bytebool::ByteBool::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_bytebool::ByteBool::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_bytebool::ByteBool::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_bytebool::ByteBool::serialize(_array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_bytebool::ByteBool::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_bytebool::ByteBool::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_bytebool::ByteBool::stats(array: &vortex_bytebool::ByteBoolData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_bytebool::ByteBool::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_bytebool::ByteBool::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_bytebool::ByteBool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -94,8 +86,6 @@ pub fn vortex_bytebool::ByteBoolData::as_slice(&self) -> &[bool] pub fn vortex_bytebool::ByteBoolData::buffer(&self) -> &vortex_array::buffer::BufferHandle -pub fn vortex_bytebool::ByteBoolData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_bytebool::ByteBoolData::from_vec>(data: alloc::vec::Vec, validity: V) -> Self pub fn vortex_bytebool::ByteBoolData::is_empty(&self) -> bool @@ -104,6 +94,8 @@ pub fn vortex_bytebool::ByteBoolData::len(&self) -> usize pub fn vortex_bytebool::ByteBoolData::new(buffer: vortex_array::buffer::BufferHandle, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_bytebool::ByteBoolData::validate(buffer: &vortex_array::buffer::BufferHandle, validity: &vortex_array::validity::Validity, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> + pub fn vortex_bytebool::ByteBoolData::validity_mask(&self) -> vortex_mask::Mask impl core::clone::Clone for vortex_bytebool::ByteBoolData @@ -118,18 +110,10 @@ impl core::convert::From>> for vortex pub fn vortex_bytebool::ByteBoolData::from(value: alloc::vec::Vec>) -> Self -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_bytebool::ByteBoolData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_bytebool::ByteBoolData pub fn vortex_bytebool::ByteBoolData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_bytebool::ByteBoolData - -pub fn vortex_bytebool::ByteBoolData::into_array(self) -> vortex_array::array::erased::ArrayRef - impl vortex_array::array::vtable::validity::ValidityHelper for vortex_bytebool::ByteBoolData pub fn vortex_bytebool::ByteBoolData::validity(&self) -> &vortex_array::validity::Validity diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index 04744355a11..48448ba7b57 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -7,9 +7,9 @@ use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; -use vortex_array::EmptyMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; @@ -19,7 +19,6 @@ use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; @@ -29,7 +28,6 @@ use vortex_array::vtable::ValidityVTableFromValidityHelper; use vortex_array::vtable::validity_to_child; use vortex_buffer::BitBuffer; use vortex_buffer::ByteBuffer; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -44,28 +42,15 @@ vtable!(ByteBool, ByteBool, ByteBoolData); impl VTable for ByteBool { type ArrayData = ByteBoolData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromValidityHelper; - fn vtable(_array: &Self::ArrayData) -> &Self { - &ByteBool - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ByteBoolData) -> usize { - array.buffer.len() - } - - fn dtype(array: &ByteBoolData) -> &DType { - &array.dtype - } - - fn stats(array: &ByteBoolData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + ByteBoolData::validate(data.buffer(), data.validity(), dtype, len) } fn array_hash(array: &ByteBoolData, state: &mut H, precision: Precision) { @@ -96,31 +81,25 @@ impl VTable for ByteBool { } } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) - } - - fn build( + &self, dtype: &DType, len: usize, - _metadata: &Self::Metadata, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + if !metadata.is_empty() { + vortex_bail!( + "ByteBoolArray expects empty metadata, got {} bytes", + metadata.len() + ); + } let validity = if children.is_empty() { Validity::from(dtype.nullability()) } else if children.len() == 1 { @@ -155,7 +134,7 @@ impl VTable for ByteBool { ); array.validity = match &slots[VALIDITY_SLOT] { Some(arr) => Validity::Array(arr.clone()), - None => Validity::from(array.dtype.nullability()), + None => Validity::from(array.validity.nullability()), }; array.slots = slots; Ok(()) @@ -194,11 +173,9 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; #[derive(Clone, Debug)] pub struct ByteBoolData { - dtype: DType, buffer: BufferHandle, validity: Validity, pub(super) slots: Vec>, - stats_set: ArrayStats, } #[derive(Clone, Debug)] @@ -207,14 +184,53 @@ pub struct ByteBool; impl ByteBool { pub const ID: ArrayId = ArrayId::new_ref("vortex.bytebool"); + pub fn new(buffer: BufferHandle, validity: Validity) -> ByteBoolArray { + let dtype = DType::Bool(validity.nullability()); + let data = ByteBoolData::new(buffer, validity); + let len = data.len(); + unsafe { Array::from_parts_unchecked(ArrayParts::new(ByteBool, dtype, len, data)) } + } + /// Construct a [`ByteBoolArray`] from a `Vec` and validity. pub fn from_vec>(data: Vec, validity: V) -> ByteBoolArray { - Array::try_from_data(ByteBoolData::from_vec(data, validity)) - .vortex_expect("ByteBoolData is always valid") + let data = ByteBoolData::from_vec(data, validity); + let dtype = DType::Bool(data.validity.nullability()); + let len = data.len(); + unsafe { Array::from_parts_unchecked(ArrayParts::new(ByteBool, dtype, len, data)) } + } + + /// Construct a [`ByteBoolArray`] from optional bools. + pub fn from_option_vec(data: Vec>) -> ByteBoolArray { + let data = ByteBoolData::from(data); + let dtype = DType::Bool(data.validity.nullability()); + let len = data.len(); + unsafe { Array::from_parts_unchecked(ArrayParts::new(ByteBool, dtype, len, data)) } } } impl ByteBoolData { + pub fn validate( + buffer: &BufferHandle, + validity: &Validity, + dtype: &DType, + len: usize, + ) -> VortexResult<()> { + let expected_dtype = DType::Bool(validity.nullability()); + vortex_ensure!( + dtype == &expected_dtype, + "expected dtype {expected_dtype}, got {dtype}" + ); + vortex_ensure!( + buffer.len() == len, + "expected len {len}, got {}", + buffer.len() + ); + if let Some(vlen) = validity.maybe_len() { + vortex_ensure!(vlen == len, "expected validity len {len}, got {vlen}"); + } + Ok(()) + } + fn make_slots(validity: &Validity, len: usize) -> Vec> { vec![validity_to_child(validity, len)] } @@ -232,11 +248,9 @@ impl ByteBoolData { } let slots = Self::make_slots(&validity, length); Self { - dtype: DType::Bool(validity.nullability()), buffer, validity, slots, - stats_set: Default::default(), } } @@ -250,11 +264,6 @@ impl ByteBoolData { self.buffer.len() == 0 } - /// Returns the logical data type of the array. - pub fn dtype(&self) -> &DType { - &self.dtype - } - /// Returns the validity mask for this array. pub fn validity_mask(&self) -> Mask { self.validity.to_mask(self.len()) @@ -319,8 +328,8 @@ mod tests { use vortex_array::ArrayContext; use vortex_array::IntoArray; use vortex_array::assert_arrays_eq; - use vortex_array::serde::ArrayParts; use vortex_array::serde::SerializeOptions; + use vortex_array::serde::SerializedArray; use vortex_array::session::ArraySession; use vortex_array::session::ArraySessionExt; use vortex_buffer::ByteBufferMut; @@ -334,8 +343,7 @@ mod tests { let v = vec![true, false]; let v_len = v.len(); - let arr = ByteBoolArray::try_from_data(ByteBoolData::from(v)) - .vortex_expect("ByteBoolData is always valid"); + let arr = ByteBool::from_vec(v, Validity::AllValid); assert_eq!(v_len, arr.len()); for idx in 0..arr.len() { @@ -343,8 +351,7 @@ mod tests { } let v = vec![Some(true), None, Some(false)]; - let arr = ByteBoolArray::try_from_data(ByteBoolData::from(v)) - .vortex_expect("ByteBoolData is always valid"); + let arr = ByteBool::from_option_vec(v); assert!(arr.is_valid(0).unwrap()); assert!(!arr.is_valid(1).unwrap()); assert!(arr.is_valid(2).unwrap()); @@ -353,8 +360,7 @@ mod tests { let v: Vec> = vec![None, None]; let v_len = v.len(); - let arr = ByteBoolArray::try_from_data(ByteBoolData::from(v)) - .vortex_expect("ByteBoolData is always valid"); + let arr = ByteBool::from_option_vec(v); assert_eq!(v_len, arr.len()); for idx in 0..arr.len() { @@ -365,13 +371,7 @@ mod tests { #[test] fn test_nullable_bytebool_serde_roundtrip() { - let array = ByteBoolArray::try_from_data(ByteBoolData::from(vec![ - Some(true), - None, - Some(false), - None, - ])) - .unwrap(); + let array = ByteBool::from_option_vec(vec![Some(true), None, Some(false), None]); let dtype = array.dtype().clone(); let len = array.len(); let session = VortexSession::empty().with::(); @@ -389,7 +389,7 @@ mod tests { concat.extend_from_slice(buf.as_ref()); } - let parts = ArrayParts::try_from(concat.freeze()).unwrap(); + let parts = SerializedArray::try_from(concat.freeze()).unwrap(); let decoded = parts .decode(&dtype, len, &ReadContext::new(ctx.to_ids()), &session) .unwrap(); diff --git a/encodings/bytebool/src/compute.rs b/encodings/bytebool/src/compute.rs index a44f30cce0c..904c8e5cf93 100644 --- a/encodings/bytebool/src/compute.rs +++ b/encodings/bytebool/src/compute.rs @@ -16,7 +16,6 @@ use vortex_array::validity::Validity; use vortex_error::VortexResult; use super::ByteBool; -use super::ByteBoolData; impl CastReduce for ByteBool { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { @@ -32,7 +31,7 @@ impl CastReduce for ByteBool { .cast_nullability(dtype.nullability(), array.len())?; return Ok(Some( - ByteBoolData::new(array.buffer().clone(), new_validity).into_array(), + ByteBool::new(array.buffer().clone(), new_validity).into_array(), )); } @@ -44,7 +43,7 @@ impl CastReduce for ByteBool { impl MaskReduce for ByteBool { fn mask(array: ArrayView<'_, Self>, mask: &ArrayRef) -> VortexResult> { Ok(Some( - ByteBoolData::new( + ByteBool::new( array.buffer().clone(), array .validity() @@ -79,9 +78,7 @@ impl TakeExecute for ByteBool { .collect::>() }); - Ok(Some( - ByteBoolData::from_vec(taken_bools, validity).into_array(), - )) + Ok(Some(ByteBool::from_vec(taken_bools, validity).into_array())) } } @@ -98,19 +95,16 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::scalar_fn::fns::operators::Operator; - use vortex_error::VortexExpect; use super::*; use crate::ByteBoolArray; fn bb(v: Vec) -> ByteBoolArray { - ByteBoolArray::try_from_data(ByteBoolData::from(v)) - .vortex_expect("ByteBoolData is always valid") + ByteBool::from_vec(v, Validity::AllValid) } fn bb_opt(v: Vec>) -> ByteBoolArray { - ByteBoolArray::try_from_data(ByteBoolData::from(v)) - .vortex_expect("ByteBoolData is always valid") + ByteBool::from_option_vec(v) } #[test] diff --git a/encodings/bytebool/src/slice.rs b/encodings/bytebool/src/slice.rs index b5926602808..4c55f5bbd40 100644 --- a/encodings/bytebool/src/slice.rs +++ b/encodings/bytebool/src/slice.rs @@ -10,12 +10,11 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::ByteBool; -use crate::ByteBoolData; impl SliceReduce for ByteBool { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { Ok(Some( - ByteBoolData::new( + ByteBool::new( array.buffer().slice(range.clone()), array.validity().slice(range)?, ) diff --git a/encodings/datetime-parts/public-api.lock b/encodings/datetime-parts/public-api.lock index 0bb4af9c59b..1daadb646fe 100644 --- a/encodings/datetime-parts/public-api.lock +++ b/encodings/datetime-parts/public-api.lock @@ -22,8 +22,6 @@ impl vortex_array::array::vtable::VTable for vortex_datetime_parts::DateTimePart pub type vortex_datetime_parts::DateTimeParts::ArrayData = vortex_datetime_parts::DateTimePartsData -pub type vortex_datetime_parts::DateTimeParts::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_datetime_parts::DateTimeParts::OperationsVTable = vortex_datetime_parts::DateTimeParts pub type vortex_datetime_parts::DateTimeParts::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -36,11 +34,7 @@ pub fn vortex_datetime_parts::DateTimeParts::buffer(_array: vortex_array::array: pub fn vortex_datetime_parts::DateTimeParts::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_datetime_parts::DateTimeParts::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_datetime_parts::DateTimeParts::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_datetime_parts::DateTimeParts::dtype(array: &vortex_datetime_parts::DateTimePartsData) -> &vortex_array::dtype::DType +pub fn vortex_datetime_parts::DateTimeParts::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_datetime_parts::DateTimeParts::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -48,23 +42,17 @@ pub fn vortex_datetime_parts::DateTimeParts::execute_parent(array: vortex_array: pub fn vortex_datetime_parts::DateTimeParts::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_datetime_parts::DateTimeParts::len(array: &vortex_datetime_parts::DateTimePartsData) -> usize - -pub fn vortex_datetime_parts::DateTimeParts::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_datetime_parts::DateTimeParts::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_datetime_parts::DateTimeParts::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_datetime_parts::DateTimeParts::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_datetime_parts::DateTimeParts::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_datetime_parts::DateTimeParts::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_datetime_parts::DateTimeParts::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_datetime_parts::DateTimeParts::stats(array: &vortex_datetime_parts::DateTimePartsData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_datetime_parts::DateTimeParts::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_datetime_parts::DateTimeParts::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_datetime_parts::DateTimeParts::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -100,34 +88,12 @@ impl vortex_array::scalar_fn::fns::mask::kernel::MaskReduce for vortex_datetime_ pub fn vortex_datetime_parts::DateTimeParts::mask(array: vortex_array::array::view::ArrayView<'_, Self>, mask: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_datetime_parts::DateTimePartsArrayParts - -pub vortex_datetime_parts::DateTimePartsArrayParts::days: vortex_array::array::erased::ArrayRef - -pub vortex_datetime_parts::DateTimePartsArrayParts::dtype: vortex_array::dtype::DType - -pub vortex_datetime_parts::DateTimePartsArrayParts::seconds: vortex_array::array::erased::ArrayRef - -pub vortex_datetime_parts::DateTimePartsArrayParts::subseconds: vortex_array::array::erased::ArrayRef - -impl core::clone::Clone for vortex_datetime_parts::DateTimePartsArrayParts - -pub fn vortex_datetime_parts::DateTimePartsArrayParts::clone(&self) -> vortex_datetime_parts::DateTimePartsArrayParts - -impl core::fmt::Debug for vortex_datetime_parts::DateTimePartsArrayParts - -pub fn vortex_datetime_parts::DateTimePartsArrayParts::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub struct vortex_datetime_parts::DateTimePartsData impl vortex_datetime_parts::DateTimePartsData pub fn vortex_datetime_parts::DateTimePartsData::days(&self) -> &vortex_array::array::erased::ArrayRef -pub fn vortex_datetime_parts::DateTimePartsData::dtype(&self) -> &vortex_array::dtype::DType - -pub fn vortex_datetime_parts::DateTimePartsData::into_parts(self) -> vortex_datetime_parts::DateTimePartsArrayParts - pub fn vortex_datetime_parts::DateTimePartsData::is_empty(&self) -> bool pub fn vortex_datetime_parts::DateTimePartsData::len(&self) -> usize @@ -136,16 +102,12 @@ pub fn vortex_datetime_parts::DateTimePartsData::seconds(&self) -> &vortex_array pub fn vortex_datetime_parts::DateTimePartsData::subseconds(&self) -> &vortex_array::array::erased::ArrayRef -pub fn vortex_datetime_parts::DateTimePartsData::try_new(dtype: vortex_array::dtype::DType, days: vortex_array::array::erased::ArrayRef, seconds: vortex_array::array::erased::ArrayRef, subseconds: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult +pub fn vortex_datetime_parts::DateTimePartsData::validate(dtype: &vortex_array::dtype::DType, days: &vortex_array::array::erased::ArrayRef, seconds: &vortex_array::array::erased::ArrayRef, subseconds: &vortex_array::array::erased::ArrayRef, len: usize) -> vortex_error::VortexResult<()> impl core::clone::Clone for vortex_datetime_parts::DateTimePartsData pub fn vortex_datetime_parts::DateTimePartsData::clone(&self) -> vortex_datetime_parts::DateTimePartsData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_datetime_parts::DateTimePartsData) -> vortex_array::array::erased::ArrayRef - impl core::convert::TryFrom for vortex_datetime_parts::DateTimePartsData pub type vortex_datetime_parts::DateTimePartsData::Error = vortex_error::VortexError @@ -156,10 +118,6 @@ impl core::fmt::Debug for vortex_datetime_parts::DateTimePartsData pub fn vortex_datetime_parts::DateTimePartsData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_datetime_parts::DateTimePartsData - -pub fn vortex_datetime_parts::DateTimePartsData::into_array(self) -> vortex_array::array::erased::ArrayRef - #[repr(C)] pub struct vortex_datetime_parts::DateTimePartsMetadata pub vortex_datetime_parts::DateTimePartsMetadata::days_ptype: i32 diff --git a/encodings/datetime-parts/src/array.rs b/encodings/datetime-parts/src/array.rs index 54cbb6528e3..f59cec3dd11 100644 --- a/encodings/datetime-parts/src/array.rs +++ b/encodings/datetime-parts/src/array.rs @@ -3,26 +3,24 @@ use std::fmt::Debug; +use prost::Message; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; -use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; -use vortex_array::SerializeMetadata; use vortex_array::arrays::TemporalArray; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityChild; @@ -74,28 +72,15 @@ impl DateTimePartsMetadata { impl VTable for DateTimeParts { type ArrayData = DateTimePartsData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; - fn vtable(_array: &Self::ArrayData) -> &Self { - &DateTimeParts - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &DateTimePartsData) -> usize { - array.days().len() - } - - fn dtype(array: &DateTimePartsData) -> &DType { - &array.dtype - } - - fn stats(array: &DateTimePartsData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + DateTimePartsData::validate(dtype, data.days(), data.seconds(), data.subseconds(), len) } fn array_hash( @@ -130,37 +115,27 @@ impl VTable for DateTimeParts { vortex_panic!("DateTimePartsArray buffer_name index {idx} out of bounds") } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(DateTimePartsMetadata { - days_ptype: PType::try_from(array.days().dtype())? as i32, - seconds_ptype: PType::try_from(array.seconds().dtype())? as i32, - subseconds_ptype: PType::try_from(array.subseconds().dtype())? as i32, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) - } - - fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata( - as DeserializeMetadata>::deserialize(bytes)?, + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some( + DateTimePartsMetadata { + days_ptype: PType::try_from(array.days().dtype())? as i32, + seconds_ptype: PType::try_from(array.seconds().dtype())? as i32, + subseconds_ptype: PType::try_from(array.subseconds().dtype())? as i32, + } + .encode_to_vec(), )) } - fn build( + fn deserialize( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = DateTimePartsMetadata::decode(metadata)?; if children.len() != 3 { vortex_bail!( "Expected 3 children for datetime-parts encoding, found {}", @@ -170,17 +145,17 @@ impl VTable for DateTimeParts { let days = children.get( 0, - &DType::Primitive(metadata.0.get_days_ptype()?, dtype.nullability()), + &DType::Primitive(metadata.get_days_ptype()?, dtype.nullability()), len, )?; let seconds = children.get( 1, - &DType::Primitive(metadata.0.get_seconds_ptype()?, Nullability::NonNullable), + &DType::Primitive(metadata.get_seconds_ptype()?, Nullability::NonNullable), len, )?; let subseconds = children.get( 2, - &DType::Primitive(metadata.0.get_subseconds_ptype()?, Nullability::NonNullable), + &DType::Primitive(metadata.get_subseconds_ptype()?, Nullability::NonNullable), len, )?; @@ -241,17 +216,7 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["days", "seconds", "subseconds #[derive(Clone, Debug)] pub struct DateTimePartsData { - dtype: DType, pub(super) slots: Vec>, - stats_set: ArrayStats, -} - -#[derive(Clone, Debug)] -pub struct DateTimePartsArrayParts { - pub dtype: DType, - pub days: ArrayRef, - pub seconds: ArrayRef, - pub subseconds: ArrayRef, } #[derive(Clone, Debug)] @@ -267,24 +232,38 @@ impl DateTimeParts { seconds: ArrayRef, subseconds: ArrayRef, ) -> VortexResult { - Array::try_from_data(DateTimePartsData::try_new( - dtype, days, seconds, subseconds, - )?) + let data = DateTimePartsData::try_new(dtype.clone(), days, seconds, subseconds)?; + let len = data.len(); + Ok( + unsafe { + Array::from_parts_unchecked(ArrayParts::new(DateTimeParts, dtype, len, data)) + }, + ) } /// Construct a [`DateTimePartsArray`] from a [`TemporalArray`]. pub fn try_from_temporal(temporal: TemporalArray) -> VortexResult { - Array::try_from_data(DateTimePartsData::try_from(temporal)?) + let dtype = temporal.dtype().clone(); + let data = DateTimePartsData::try_from(temporal)?; + let len = data.len(); + Ok( + unsafe { + Array::from_parts_unchecked(ArrayParts::new(DateTimeParts, dtype, len, data)) + }, + ) } } impl DateTimePartsData { - pub fn try_new( - dtype: DType, - days: ArrayRef, - seconds: ArrayRef, - subseconds: ArrayRef, - ) -> VortexResult { + pub fn validate( + dtype: &DType, + days: &ArrayRef, + seconds: &ArrayRef, + subseconds: &ArrayRef, + len: usize, + ) -> VortexResult<()> { + vortex_ensure!(days.len() == len, "expected len {len}, got {}", days.len()); + if !days.dtype().is_int() || (dtype.is_nullable() != days.dtype().is_nullable()) { vortex_bail!( "Expected integer with nullability {}, got {}", @@ -299,8 +278,7 @@ impl DateTimePartsData { vortex_bail!(MismatchedTypes: "non-nullable integer", subseconds.dtype()); } - let length = days.len(); - if length != seconds.len() || length != subseconds.len() { + if len != seconds.len() || len != subseconds.len() { vortex_bail!( "Mismatched lengths {} {} {}", days.len(), @@ -309,39 +287,19 @@ impl DateTimePartsData { ); } - Ok(Self { - dtype, - slots: vec![Some(days), Some(seconds), Some(subseconds)], - stats_set: Default::default(), - }) + Ok(()) } - pub(crate) unsafe fn new_unchecked( + pub(crate) fn try_new( dtype: DType, days: ArrayRef, seconds: ArrayRef, subseconds: ArrayRef, - ) -> Self { - Self { - dtype, + ) -> VortexResult { + Self::validate(&dtype, &days, &seconds, &subseconds, days.len())?; + Ok(Self { slots: vec![Some(days), Some(seconds), Some(subseconds)], - stats_set: Default::default(), - } - } - - pub fn into_parts(mut self) -> DateTimePartsArrayParts { - DateTimePartsArrayParts { - dtype: self.dtype, - days: self.slots[DAYS_SLOT] - .take() - .vortex_expect("DateTimePartsArray days slot"), - seconds: self.slots[SECONDS_SLOT] - .take() - .vortex_expect("DateTimePartsArray seconds slot"), - subseconds: self.slots[SUBSECONDS_SLOT] - .take() - .vortex_expect("DateTimePartsArray subseconds slot"), - } + }) } /// Returns the number of elements in the array. @@ -354,11 +312,6 @@ impl DateTimePartsData { self.days().len() == 0 } - /// Returns the logical data type of the array. - pub fn dtype(&self) -> &DType { - &self.dtype - } - pub fn days(&self) -> &ArrayRef { self.slots[DAYS_SLOT] .as_ref() diff --git a/encodings/datetime-parts/src/canonical.rs b/encodings/datetime-parts/src/canonical.rs index 99337f898a5..daecaecfd3f 100644 --- a/encodings/datetime-parts/src/canonical.rs +++ b/encodings/datetime-parts/src/canonical.rs @@ -119,8 +119,7 @@ mod test { use vortex_error::VortexResult; use vortex_session::VortexSession; - use crate::DateTimePartsArray; - use crate::DateTimePartsData; + use crate::DateTimeParts; use crate::canonical::decode_to_temporal; #[rstest] @@ -140,14 +139,11 @@ mod test { ], validity.clone(), ); - let date_times = DateTimePartsArray::try_from_data( - DateTimePartsData::try_from(TemporalArray::new_timestamp( - milliseconds.clone().into_array(), - TimeUnit::Milliseconds, - Some("UTC".into()), - )) - .unwrap(), - )?; + let date_times = DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( + milliseconds.clone().into_array(), + TimeUnit::Milliseconds, + Some("UTC".into()), + ))?; let mut ctx = ExecutionCtx::new(VortexSession::empty()); diff --git a/encodings/datetime-parts/src/compute/cast.rs b/encodings/datetime-parts/src/compute/cast.rs index 9c0c405827d..ca846bbd424 100644 --- a/encodings/datetime-parts/src/compute/cast.rs +++ b/encodings/datetime-parts/src/compute/cast.rs @@ -10,7 +10,6 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::DateTimeParts; -use crate::DateTimePartsData; impl CastReduce for DateTimeParts { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { if !array.dtype().eq_ignore_nullability(dtype) { @@ -18,7 +17,7 @@ impl CastReduce for DateTimeParts { }; Ok(Some( - DateTimePartsData::try_new( + DateTimeParts::try_new( dtype.clone(), array .days() @@ -45,11 +44,11 @@ mod tests { use vortex_array::validity::Validity; use vortex_buffer::buffer; + use crate::DateTimeParts; use crate::DateTimePartsArray; - use crate::DateTimePartsData; fn date_time_array(validity: Validity) -> ArrayRef { - DateTimePartsData::try_from(TemporalArray::new_timestamp( + DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( PrimitiveArray::new( buffer![ 86_400i64, // element with only day component @@ -104,7 +103,7 @@ mod tests { } #[rstest] - #[case(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( buffer![ 0i64, 86_400_000, // 1 day in ms @@ -114,8 +113,8 @@ mod tests { ].into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap()).unwrap())] - #[case(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + )).unwrap())] + #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( PrimitiveArray::from_option_iter([ Some(0i64), None, @@ -125,12 +124,12 @@ mod tests { ]).into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap()).unwrap())] - #[case(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + )).unwrap())] + #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( buffer![86_400_000_000_000i64].into_array(), // 1 day in ns TimeUnit::Nanoseconds, Some("UTC".into()) - )).unwrap()).unwrap())] + )).unwrap())] fn test_cast_datetime_parts_conformance(#[case] array: DateTimePartsArray) { use vortex_array::compute::conformance::cast::test_cast_conformance; test_cast_conformance(&array.into_array()); diff --git a/encodings/datetime-parts/src/compute/compare.rs b/encodings/datetime-parts/src/compute/compare.rs index 0c92e6a5168..76250bd320e 100644 --- a/encodings/datetime-parts/src/compute/compare.rs +++ b/encodings/datetime-parts/src/compute/compare.rs @@ -205,13 +205,14 @@ mod test { use vortex_buffer::buffer; use super::*; - use crate::DateTimePartsData; + use crate::DateTimeParts; + use crate::DateTimePartsArray; fn dtp_array_from_timestamp( value: T, validity: Validity, - ) -> DateTimePartsData { - DateTimePartsData::try_from(TemporalArray::new_timestamp( + ) -> DateTimePartsArray { + DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( PrimitiveArray::new(buffer![value], validity).into_array(), TimeUnit::Seconds, Some("UTC".into()), @@ -312,7 +313,7 @@ mod test { Some("UTC".into()), ); - let lhs = DateTimePartsData::try_new( + let lhs = DateTimeParts::try_new( DType::Extension(temporal_array.ext_dtype()), PrimitiveArray::new(buffer![0i32], lhs_validity).into_array(), PrimitiveArray::new(buffer![0u32], Validity::NonNullable).into_array(), diff --git a/encodings/datetime-parts/src/compute/filter.rs b/encodings/datetime-parts/src/compute/filter.rs index ba339063540..da825d8a95e 100644 --- a/encodings/datetime-parts/src/compute/filter.rs +++ b/encodings/datetime-parts/src/compute/filter.rs @@ -9,11 +9,10 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::DateTimeParts; -use crate::DateTimePartsData; impl FilterReduce for DateTimeParts { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { Ok(Some( - DateTimePartsData::try_new( + DateTimeParts::try_new( array.dtype().clone(), array.days().filter(mask.clone())?, array.seconds().filter(mask.clone())?, @@ -33,7 +32,7 @@ mod test { use vortex_array::extension::datetime::TimeUnit; use vortex_buffer::buffer; - use crate::DateTimePartsData; + use crate::DateTimeParts; #[test] fn test_filter_datetime_parts() { @@ -50,7 +49,7 @@ mod test { let temporal = TemporalArray::new_timestamp(timestamps, TimeUnit::Milliseconds, Some("UTC".into())); - let array = DateTimePartsData::try_from(temporal).unwrap(); + let array = DateTimeParts::try_from_temporal(temporal).unwrap(); test_filter_conformance(&array.into_array()); // Test with nullable values @@ -66,7 +65,7 @@ mod test { let temporal = TemporalArray::new_timestamp(timestamps, TimeUnit::Milliseconds, Some("UTC".into())); - let array = DateTimePartsData::try_from(temporal).unwrap(); + let array = DateTimeParts::try_from_temporal(temporal).unwrap(); test_filter_conformance(&array.into_array()); } } diff --git a/encodings/datetime-parts/src/compute/mask.rs b/encodings/datetime-parts/src/compute/mask.rs index 023831d251a..290707ef8ef 100644 --- a/encodings/datetime-parts/src/compute/mask.rs +++ b/encodings/datetime-parts/src/compute/mask.rs @@ -9,13 +9,12 @@ use vortex_array::scalar_fn::fns::mask::MaskReduce; use vortex_error::VortexResult; use crate::DateTimeParts; -use crate::DateTimePartsData; impl MaskReduce for DateTimeParts { fn mask(array: ArrayView<'_, Self>, mask: &ArrayRef) -> VortexResult> { let masked_days = array.days().clone().mask(mask.clone())?; Ok(Some( - DateTimePartsData::try_new( + DateTimeParts::try_new( array.dtype().as_nullable(), masked_days, array.seconds().clone(), diff --git a/encodings/datetime-parts/src/compute/mod.rs b/encodings/datetime-parts/src/compute/mod.rs index 4ca8398fd6b..53717a52b0c 100644 --- a/encodings/datetime-parts/src/compute/mod.rs +++ b/encodings/datetime-parts/src/compute/mod.rs @@ -21,55 +21,59 @@ mod tests { use vortex_array::extension::datetime::TimeUnit; use vortex_buffer::buffer; + use crate::DateTimeParts; use crate::DateTimePartsArray; - use crate::DateTimePartsData; + + fn dtp_from_temporal(temporal: TemporalArray) -> DateTimePartsArray { + DateTimeParts::try_from_temporal(temporal).unwrap() + } #[rstest] // Basic datetime arrays - #[case::datetime_seconds(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + #[case::datetime_seconds(dtp_from_temporal(TemporalArray::new_timestamp( buffer![0i64, 86400, 172800, 259200, 345600].into_array(), TimeUnit::Seconds, Some("UTC".into()), - )).unwrap()).unwrap())] - #[case::datetime_millis(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + )))] + #[case::datetime_millis(dtp_from_temporal(TemporalArray::new_timestamp( buffer![0i64, 86400000, 172800000].into_array(), TimeUnit::Milliseconds, Some("UTC".into()), - )).unwrap()).unwrap())] - #[case::datetime_micros(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + )))] + #[case::datetime_micros(dtp_from_temporal(TemporalArray::new_timestamp( buffer![0i64, 86400000000, 172800000000].into_array(), TimeUnit::Microseconds, Some("UTC".into()), - )).unwrap()).unwrap())] - #[case::datetime_nanos(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + )))] + #[case::datetime_nanos(dtp_from_temporal(TemporalArray::new_timestamp( buffer![0i64, 86400000000000].into_array(), TimeUnit::Nanoseconds, Some("UTC".into()), - )).unwrap()).unwrap())] + )))] // Nullable arrays - #[case::datetime_nullable_seconds(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + #[case::datetime_nullable_seconds(dtp_from_temporal(TemporalArray::new_timestamp( PrimitiveArray::from_option_iter([Some(0i64), None, Some(86400), Some(172800), None]).into_array(), TimeUnit::Seconds, Some("UTC".into()), - )).unwrap()).unwrap())] + )))] // Edge cases - #[case::datetime_single(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + #[case::datetime_single(dtp_from_temporal(TemporalArray::new_timestamp( buffer![1234567890i64].into_array(), TimeUnit::Seconds, Some("UTC".into()), - )).unwrap()).unwrap())] + )))] // Large arrays (> 1024 elements) - #[case::datetime_large(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + #[case::datetime_large(dtp_from_temporal(TemporalArray::new_timestamp( PrimitiveArray::from_iter((0..1500).map(|i| i as i64 * 86400)).into_array(), TimeUnit::Seconds, Some("UTC".into()), - )).unwrap()).unwrap())] + )))] // Different time patterns - #[case::datetime_with_subseconds(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + #[case::datetime_with_subseconds(dtp_from_temporal(TemporalArray::new_timestamp( buffer![123456789i64, 234567890, 345678901, 456789012, 567890123].into_array(), TimeUnit::Milliseconds, Some("UTC".into()), - )).unwrap()).unwrap())] + )))] fn test_datetime_parts_consistency(#[case] array: DateTimePartsArray) { test_array_consistency(&array.into_array()); diff --git a/encodings/datetime-parts/src/compute/rules.rs b/encodings/datetime-parts/src/compute/rules.rs index dedc87caa13..92c1f877c08 100644 --- a/encodings/datetime-parts/src/compute/rules.rs +++ b/encodings/datetime-parts/src/compute/rules.rs @@ -26,7 +26,6 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::DateTimeParts; -use crate::DateTimePartsData; use crate::timestamp; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&DTPFilterPushDownRule), @@ -57,7 +56,7 @@ impl ArrayParentReduceRule for DTPFilterPushDownRule { return Ok(None); } - DateTimePartsData::try_new( + DateTimeParts::try_new( child.dtype().clone(), child.days().clone().filter(parent.filter_mask().clone())?, ConstantArray::new( @@ -192,6 +191,7 @@ mod tests { use vortex_buffer::Buffer; use super::*; + use crate::DateTimeParts; use crate::DateTimePartsArray; const SECONDS_PER_DAY: i64 = 86400; @@ -215,8 +215,8 @@ mod tests { time_unit, None, ); - DateTimePartsArray::try_from_data(DateTimePartsData::try_from(temporal).unwrap()) - .vortex_expect("DateTimePartsData is always valid") + DateTimeParts::try_from_temporal(temporal) + .vortex_expect("TemporalArray must produce valid DateTimeParts") } /// Create a constant timestamp scalar at midnight for the given day. @@ -348,7 +348,7 @@ mod tests { TimeUnit::Seconds, None, ); - let dtp = DateTimePartsData::try_from(temporal).unwrap(); + let dtp = DateTimeParts::try_from_temporal(temporal).unwrap(); let len = dtp.len(); // Compare against midnight constant diff --git a/encodings/datetime-parts/src/compute/slice.rs b/encodings/datetime-parts/src/compute/slice.rs index b63693980f7..5bad980d8dc 100644 --- a/encodings/datetime-parts/src/compute/slice.rs +++ b/encodings/datetime-parts/src/compute/slice.rs @@ -10,19 +10,18 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::DateTimeParts; -use crate::DateTimePartsData; impl SliceReduce for DateTimeParts { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { // SAFETY: slicing all components preserves values - Ok(Some(unsafe { - DateTimePartsData::new_unchecked( + Ok(Some( + DateTimeParts::try_new( array.dtype().clone(), array.days().slice(range.clone())?, array.seconds().slice(range.clone())?, array.subseconds().slice(range)?, - ) - .into_array() - })) + )? + .into_array(), + )) } } diff --git a/encodings/datetime-parts/src/compute/take.rs b/encodings/datetime-parts/src/compute/take.rs index b804752657b..4cd214663c1 100644 --- a/encodings/datetime-parts/src/compute/take.rs +++ b/encodings/datetime-parts/src/compute/take.rs @@ -16,7 +16,6 @@ use vortex_error::VortexResult; use vortex_error::vortex_panic; use crate::DateTimeParts; -use crate::DateTimePartsData; fn take_datetime_parts( array: ArrayView, indices: &ArrayRef, @@ -39,7 +38,7 @@ fn take_datetime_parts( if !taken_seconds.dtype().is_nullable() && !taken_subseconds.dtype().is_nullable() { return Ok( - DateTimePartsData::try_new(dtype, taken_days, taken_seconds, taken_subseconds)? + DateTimeParts::try_new(dtype, taken_days, taken_seconds, taken_subseconds)? .into_array(), ); } @@ -78,10 +77,7 @@ fn take_datetime_parts( .cast(array.subseconds().dtype())?; let taken_subseconds = taken_subseconds.fill_null(subseconds_fill)?; - Ok( - DateTimePartsData::try_new(dtype, taken_days, taken_seconds, taken_subseconds)? - .into_array(), - ) + Ok(DateTimeParts::try_new(dtype, taken_days, taken_seconds, taken_subseconds)?.into_array()) } impl TakeExecute for DateTimeParts { @@ -104,11 +100,11 @@ mod tests { use vortex_array::extension::datetime::TimeUnit; use vortex_buffer::buffer; + use crate::DateTimeParts; use crate::DateTimePartsArray; - use crate::DateTimePartsData; #[rstest] - #[case(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( buffer![ 0i64, 86_400_000, // 1 day in ms @@ -118,8 +114,8 @@ mod tests { ].into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap()).unwrap())] - #[case(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + )).unwrap())] + #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( PrimitiveArray::from_option_iter([ Some(0i64), None, @@ -129,12 +125,12 @@ mod tests { ]).into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap()).unwrap())] - #[case(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( + )).unwrap())] + #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( buffer![86_400_000i64].into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap()).unwrap())] + )).unwrap())] fn test_take_datetime_parts_conformance(#[case] array: DateTimePartsArray) { test_take_conformance(&array.into_array()); } diff --git a/encodings/datetime-parts/src/lib.rs b/encodings/datetime-parts/src/lib.rs index 93032b0b97d..091f4058c14 100644 --- a/encodings/datetime-parts/src/lib.rs +++ b/encodings/datetime-parts/src/lib.rs @@ -30,7 +30,7 @@ pub fn initialize(session: &mut VortexSession) { #[cfg(test)] mod test { - use vortex_array::ProstMetadata; + use prost::Message; use vortex_array::dtype::PType; use vortex_array::test_harness::check_metadata; @@ -41,11 +41,12 @@ mod test { fn test_datetimeparts_metadata() { check_metadata( "datetimeparts.metadata", - ProstMetadata(DateTimePartsMetadata { + &DateTimePartsMetadata { days_ptype: PType::I64 as i32, seconds_ptype: PType::I64 as i32, subseconds_ptype: PType::I64 as i32, - }), + } + .encode_to_vec(), ); } } diff --git a/encodings/decimal-byte-parts/public-api.lock b/encodings/decimal-byte-parts/public-api.lock index 1a93f2602f4..91d35435cab 100644 --- a/encodings/decimal-byte-parts/public-api.lock +++ b/encodings/decimal-byte-parts/public-api.lock @@ -20,8 +20,6 @@ impl vortex_array::array::vtable::VTable for vortex_decimal_byte_parts::DecimalB pub type vortex_decimal_byte_parts::DecimalByteParts::ArrayData = vortex_decimal_byte_parts::DecimalBytePartsData -pub type vortex_decimal_byte_parts::DecimalByteParts::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_decimal_byte_parts::DecimalByteParts::OperationsVTable = vortex_decimal_byte_parts::DecimalByteParts pub type vortex_decimal_byte_parts::DecimalByteParts::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -34,11 +32,7 @@ pub fn vortex_decimal_byte_parts::DecimalByteParts::buffer(_array: vortex_array: pub fn vortex_decimal_byte_parts::DecimalByteParts::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_decimal_byte_parts::DecimalByteParts::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_decimal_byte_parts::DecimalByteParts::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_decimal_byte_parts::DecimalByteParts::dtype(array: &vortex_decimal_byte_parts::DecimalBytePartsData) -> &vortex_array::dtype::DType +pub fn vortex_decimal_byte_parts::DecimalByteParts::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_decimal_byte_parts::DecimalByteParts::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -46,23 +40,17 @@ pub fn vortex_decimal_byte_parts::DecimalByteParts::execute_parent(array: vortex pub fn vortex_decimal_byte_parts::DecimalByteParts::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_decimal_byte_parts::DecimalByteParts::len(array: &vortex_decimal_byte_parts::DecimalBytePartsData) -> usize - -pub fn vortex_decimal_byte_parts::DecimalByteParts::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_decimal_byte_parts::DecimalByteParts::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_decimal_byte_parts::DecimalByteParts::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_decimal_byte_parts::DecimalByteParts::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_decimal_byte_parts::DecimalByteParts::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_decimal_byte_parts::DecimalByteParts::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_decimal_byte_parts::DecimalByteParts::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_decimal_byte_parts::DecimalByteParts::stats(array: &vortex_decimal_byte_parts::DecimalBytePartsData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_decimal_byte_parts::DecimalByteParts::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_decimal_byte_parts::DecimalByteParts::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_decimal_byte_parts::DecimalByteParts::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -98,43 +86,29 @@ impl vortex_array::scalar_fn::fns::mask::kernel::MaskReduce for vortex_decimal_b pub fn vortex_decimal_byte_parts::DecimalByteParts::mask(array: vortex_array::array::view::ArrayView<'_, Self>, mask: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_decimal_byte_parts::DecimalBytePartsArrayParts - -pub vortex_decimal_byte_parts::DecimalBytePartsArrayParts::dtype: vortex_array::dtype::DType - -pub vortex_decimal_byte_parts::DecimalBytePartsArrayParts::msp: vortex_array::array::erased::ArrayRef - pub struct vortex_decimal_byte_parts::DecimalBytePartsData impl vortex_decimal_byte_parts::DecimalBytePartsData -pub fn vortex_decimal_byte_parts::DecimalBytePartsData::decimal_dtype(&self) -> &vortex_array::dtype::decimal::DecimalDType - -pub fn vortex_decimal_byte_parts::DecimalBytePartsData::dtype(&self) -> &vortex_array::dtype::DType - -pub fn vortex_decimal_byte_parts::DecimalBytePartsData::into_parts(self) -> vortex_decimal_byte_parts::DecimalBytePartsArrayParts +pub fn vortex_decimal_byte_parts::DecimalBytePartsData::into_parts(self) -> vortex_decimal_byte_parts::DecimalBytePartsDataParts pub fn vortex_decimal_byte_parts::DecimalBytePartsData::is_empty(&self) -> bool pub fn vortex_decimal_byte_parts::DecimalBytePartsData::len(&self) -> usize -pub fn vortex_decimal_byte_parts::DecimalBytePartsData::try_new(msp: vortex_array::array::erased::ArrayRef, decimal_dtype: vortex_array::dtype::decimal::DecimalDType) -> vortex_error::VortexResult +pub fn vortex_decimal_byte_parts::DecimalBytePartsData::validate(msp: &vortex_array::array::erased::ArrayRef, decimal_dtype: vortex_array::dtype::decimal::DecimalDType, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> impl core::clone::Clone for vortex_decimal_byte_parts::DecimalBytePartsData pub fn vortex_decimal_byte_parts::DecimalBytePartsData::clone(&self) -> vortex_decimal_byte_parts::DecimalBytePartsData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_decimal_byte_parts::DecimalBytePartsData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_decimal_byte_parts::DecimalBytePartsData pub fn vortex_decimal_byte_parts::DecimalBytePartsData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_decimal_byte_parts::DecimalBytePartsData +pub struct vortex_decimal_byte_parts::DecimalBytePartsDataParts -pub fn vortex_decimal_byte_parts::DecimalBytePartsData::into_array(self) -> vortex_array::array::erased::ArrayRef +pub vortex_decimal_byte_parts::DecimalBytePartsDataParts::msp: vortex_array::array::erased::ArrayRef pub struct vortex_decimal_byte_parts::DecimalBytesPartsMetadata diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs index 5314dd8845d..d78c6e2a96d 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs @@ -7,6 +7,7 @@ use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::DecimalByteParts; @@ -19,7 +20,11 @@ impl CastReduce for DecimalByteParts { }; // Check if this is just a nullability change - if array.decimal_dtype() == target_decimal + if array + .dtype() + .as_decimal_opt() + .vortex_expect("must be a decimal dtype") + == target_decimal && array.dtype().nullability() != *target_nullability { // Cast the msp array to handle nullability change diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs index b1be5b1b54d..b0afad10cfa 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs @@ -38,7 +38,7 @@ impl CompareKernel for DecimalByteParts { return Ok(None); }; - let nullability = lhs.dtype.nullability() | rhs.dtype().nullability(); + let nullability = lhs.dtype().nullability() | rhs.dtype().nullability(); let scalar_type = lhs.msp().dtype().with_nullability(nullability); let rhs_decimal = rhs_const diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs index 638c167e9ed..49435845476 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs @@ -5,14 +5,21 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::arrays::filter::FilterReduce; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_mask::Mask; use crate::DecimalByteParts; impl FilterReduce for DecimalByteParts { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { - DecimalByteParts::try_new(array.msp().filter(mask.clone())?, *array.decimal_dtype()) - .map(|d| Some(d.into_array())) + DecimalByteParts::try_new( + array.msp().filter(mask.clone())?, + *array + .dtype() + .as_decimal_opt() + .vortex_expect("must be a decimal dtype"), + ) + .map(|d| Some(d.into_array())) } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs index f3d5068faa0..4c1b1a36c64 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs @@ -8,9 +8,9 @@ use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; use vortex_array::scalar_fn::EmptyOptions; use vortex_array::scalar_fn::fns::mask::Mask as MaskExpr; use vortex_array::scalar_fn::fns::mask::MaskReduce; +use vortex_error::VortexExpect; use vortex_error::VortexResult; -use super::DecimalBytePartsData; use crate::DecimalByteParts; impl MaskReduce for DecimalByteParts { @@ -21,7 +21,14 @@ impl MaskReduce for DecimalByteParts { [array.msp().clone(), mask.clone()], )?; Ok(Some( - DecimalBytePartsData::try_new(masked_msp, *array.decimal_dtype())?.into_array(), + DecimalByteParts::try_new( + masked_msp, + *array + .dtype() + .as_decimal_opt() + .vortex_expect("must be a decimal dtype"), + )? + .into_array(), )) } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mod.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mod.rs index ef271694aea..35da5be40b6 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mod.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mod.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use super::DecimalBytePartsData; mod cast; mod compare; mod filter; diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs index 9aa64aa8d37..3c7f7ecfabf 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs @@ -6,9 +6,9 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::dict::TakeExecute; +use vortex_error::VortexExpect; use vortex_error::VortexResult; -use super::DecimalBytePartsData; use crate::DecimalByteParts; impl TakeExecute for DecimalByteParts { @@ -17,7 +17,13 @@ impl TakeExecute for DecimalByteParts { indices: &ArrayRef, _ctx: &mut ExecutionCtx, ) -> VortexResult> { - DecimalBytePartsData::try_new(array.msp().take(indices.clone())?, *array.decimal_dtype()) - .map(|a| Some(a.into_array())) + DecimalByteParts::try_new( + array.msp().take(indices.clone())?, + *array + .dtype() + .as_decimal_opt() + .vortex_expect("must be a decimal dtype"), + ) + .map(|a| Some(a.into_array())) } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs index 5fff2a47bf5..8773b2e8184 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::Array; +use vortex_array::ArrayParts; use vortex_array::ArrayView; pub(crate) mod compute; mod rules; @@ -16,8 +17,6 @@ use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; -use vortex_array::SerializeMetadata; use vortex_array::arrays::DecimalArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; @@ -29,7 +28,6 @@ use vortex_array::scalar::DecimalValue; use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; use vortex_array::vtable::VTable; @@ -58,28 +56,18 @@ pub struct DecimalBytesPartsMetadata { impl VTable for DecimalByteParts { type ArrayData = DecimalBytePartsData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; - fn vtable(_array: &Self::ArrayData) -> &Self { - &DecimalByteParts - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &DecimalBytePartsData) -> usize { - array.msp().len() - } - - fn dtype(array: &DecimalBytePartsData) -> &DType { - &array.dtype - } - - fn stats(array: &DecimalBytePartsData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + let Some(decimal_dtype) = dtype.as_decimal_opt() else { + vortex_bail!("expected decimal dtype, got {}", dtype) + }; + DecimalBytePartsData::validate(data.msp(), *decimal_dtype, dtype, len) } fn array_hash( @@ -110,34 +98,26 @@ impl VTable for DecimalByteParts { vortex_panic!("DecimalBytePartsArray buffer_name index {idx} out of bounds") } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(DecimalBytesPartsMetadata { - zeroth_child_ptype: PType::try_from(array.msp().dtype())? as i32, - lower_part_count: 0, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some( + DecimalBytesPartsMetadata { + zeroth_child_ptype: PType::try_from(array.msp().dtype())? as i32, + lower_part_count: 0, + } + .encode_to_vec(), + )) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata(DecimalBytesPartsMetadata::decode(bytes)?)) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = DecimalBytesPartsMetadata::decode(metadata)?; let Some(decimal_dtype) = dtype.as_decimal_opt() else { vortex_bail!("decoding decimal but given non decimal dtype {}", dtype) }; @@ -211,50 +191,40 @@ pub struct DecimalBytePartsData { // NOTE: the lower_parts is currently unused, we reserve this field so that it is properly // read/written during serde, but provide no constructor to initialize this to anything // other than the empty Vec. - // Must update `DecimalBytePartsArrayParts` too. _lower_parts: Vec, - dtype: DType, - stats_set: ArrayStats, } -pub struct DecimalBytePartsArrayParts { +pub struct DecimalBytePartsDataParts { pub msp: ArrayRef, - pub dtype: DType, } impl DecimalBytePartsData { - pub fn try_new(msp: ArrayRef, decimal_dtype: DecimalDType) -> VortexResult { + pub fn validate( + msp: &ArrayRef, + decimal_dtype: DecimalDType, + dtype: &DType, + len: usize, + ) -> VortexResult<()> { if !msp.dtype().is_signed_int() { vortex_bail!("decimal bytes parts, first part must be a signed array") } - let nullable = msp.dtype().nullability(); - Ok(Self { - slots: vec![Some(msp)], - _lower_parts: Vec::new(), - dtype: DType::Decimal(decimal_dtype, nullable), - stats_set: Default::default(), - }) + let expected_dtype = DType::Decimal(decimal_dtype, msp.dtype().nullability()); + vortex_ensure!( + dtype == &expected_dtype, + "expected dtype {expected_dtype}, got {dtype}" + ); + vortex_ensure!(msp.len() == len, "expected len {len}, got {}", msp.len()); + Ok(()) } - pub(crate) unsafe fn new_unchecked(msp: ArrayRef, decimal_dtype: DecimalDType) -> Self { - let nullable = msp.dtype().nullability(); - Self { + pub(crate) fn try_new(msp: ArrayRef, decimal_dtype: DecimalDType) -> VortexResult { + let dtype = DType::Decimal(decimal_dtype, msp.dtype().nullability()); + Self::validate(&msp, decimal_dtype, &dtype, msp.len())?; + Ok(Self { slots: vec![Some(msp)], _lower_parts: Vec::new(), - dtype: DType::Decimal(decimal_dtype, nullable), - stats_set: Default::default(), - } - } - - /// If `_lower_parts` is supported check all calls use this correctly. - pub fn into_parts(mut self) -> DecimalBytePartsArrayParts { - DecimalBytePartsArrayParts { - msp: self.slots[MSP_SLOT] - .take() - .vortex_expect("DecimalBytePartsArray msp slot"), - dtype: self.dtype, - } + }) } /// Returns the number of elements in the array. @@ -267,22 +237,19 @@ impl DecimalBytePartsData { self.msp().len() == 0 } - /// Returns the logical data type of the array. - pub fn dtype(&self) -> &DType { - &self.dtype - } - - pub fn decimal_dtype(&self) -> &DecimalDType { - self.dtype - .as_decimal_opt() - .vortex_expect("must be a decimal dtype") - } - pub(crate) fn msp(&self) -> &ArrayRef { self.slots[MSP_SLOT] .as_ref() .vortex_expect("DecimalBytePartsArray msp slot") } + + pub fn into_parts(mut self) -> DecimalBytePartsDataParts { + DecimalBytePartsDataParts { + msp: self.slots[MSP_SLOT] + .take() + .vortex_expect("DecimalBytePartsArray msp slot"), + } + } } #[derive(Clone, Debug)] @@ -296,7 +263,12 @@ impl DecimalByteParts { msp: ArrayRef, decimal_dtype: DecimalDType, ) -> VortexResult { - Array::try_from_data(DecimalBytePartsData::try_new(msp, decimal_dtype)?) + let data = DecimalBytePartsData::try_new(msp, decimal_dtype)?; + let dtype = DType::Decimal(decimal_dtype, data.msp().dtype().nullability()); + let len = data.len(); + Ok(unsafe { + Array::from_parts_unchecked(ArrayParts::new(DecimalByteParts, dtype, len, data)) + }) } } @@ -316,7 +288,10 @@ fn to_canonical_decimal( unsafe { DecimalArray::new_unchecked( prim.to_buffer::

(), - *array.decimal_dtype(), + *array + .dtype() + .as_decimal_opt() + .vortex_expect("must be a decimal dtype"), prim.validity(), ) } @@ -338,7 +313,7 @@ impl OperationsVTable for DecimalByteParts { // TODO(joe): extend this to support multiple parts. let value = primitive_scalar.as_::().vortex_expect("non-null"); Scalar::try_new( - array.dtype.clone(), + array.dtype().clone(), Some(ScalarValue::Decimal(DecimalValue::I64(value))), ) } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs index f9b4422f7a6..0eeaeed5716 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs @@ -11,9 +11,9 @@ use vortex_array::optimizer::rules::ArrayParentReduceRule; use vortex_array::optimizer::rules::ParentRuleSet; use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; use vortex_array::scalar_fn::fns::mask::MaskReduceAdaptor; +use vortex_error::VortexExpect; use vortex_error::VortexResult; -use super::DecimalBytePartsData; use crate::DecimalByteParts; pub(super) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ @@ -43,8 +43,14 @@ impl ArrayParentReduceRule for DecimalBytePartsFilterPushDownR } let new_msp = child.msp().filter(parent.filter_mask().clone())?; - let new_child = - DecimalBytePartsData::try_new(new_msp, *child.decimal_dtype())?.into_array(); + let new_child = DecimalByteParts::try_new( + new_msp, + *child + .dtype() + .as_decimal_opt() + .vortex_expect("must be a decimal dtype"), + )? + .into_array(); Ok(Some(new_child)) } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/slice.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/slice.rs index 051d80ff016..89275570a53 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/slice.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/slice.rs @@ -7,17 +7,22 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::arrays::slice::SliceReduce; +use vortex_error::VortexExpect; use vortex_error::VortexResult; -use super::DecimalBytePartsData; use crate::DecimalByteParts; impl SliceReduce for DecimalByteParts { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - // SAFETY: slicing encoded MSP does not change the encoded values - Ok(Some(unsafe { - DecimalBytePartsData::new_unchecked(array.msp().slice(range)?, *array.decimal_dtype()) - .into_array() - })) + Ok(Some( + DecimalByteParts::try_new( + array.msp().slice(range)?, + *array + .dtype() + .as_decimal_opt() + .vortex_expect("must be a decimal dtype"), + )? + .into_array(), + )) } } diff --git a/encodings/fastlanes/benches/bitpacking_take.rs b/encodings/fastlanes/benches/bitpacking_take.rs index e0325b7a481..6129f479d47 100644 --- a/encodings/fastlanes/benches/bitpacking_take.rs +++ b/encodings/fastlanes/benches/bitpacking_take.rs @@ -160,9 +160,9 @@ fn patched_take_10_stratified(bencher: Bencher) { let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); - assert!(packed.patches().is_some()); + assert!(packed.patches(packed.len()).is_some()); assert_eq!( - packed.patches().unwrap().num_patches(), + packed.patches(packed.len()).unwrap().num_patches(), NUM_EXCEPTIONS as usize ); @@ -185,9 +185,9 @@ fn patched_take_10_contiguous(bencher: Bencher) { let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); - assert!(packed.patches().is_some()); + assert!(packed.patches(packed.len()).is_some()); assert_eq!( - packed.patches().unwrap().num_patches(), + packed.patches(packed.len()).unwrap().num_patches(), NUM_EXCEPTIONS as usize ); @@ -249,9 +249,9 @@ fn patched_take_10k_contiguous_patches(bencher: Bencher) { let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); - assert!(packed.patches().is_some()); + assert!(packed.patches(packed.len()).is_some()); assert_eq!( - packed.patches().unwrap().num_patches(), + packed.patches(packed.len()).unwrap().num_patches(), NUM_EXCEPTIONS as usize ); diff --git a/encodings/fastlanes/goldenfiles/for.metadata b/encodings/fastlanes/goldenfiles/for.metadata new file mode 100644 index 00000000000..0e9da711b3e --- /dev/null +++ b/encodings/fastlanes/goldenfiles/for.metadata @@ -0,0 +1 @@ +þÿÿÿÿÿÿÿÿ \ No newline at end of file diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index 606dae09f3e..4f426089f50 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -40,11 +40,11 @@ pub fn vortex_fastlanes::bitpack_decompress::apply_patches_to_uninit_range_fn usize -pub fn vortex_fastlanes::bitpack_decompress::unpack_array(array: &vortex_fastlanes::BitPackedData, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fastlanes::bitpack_decompress::unpack_array(array: vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_fastlanes::bitpack_decompress::unpack_primitive_array(array: &vortex_fastlanes::BitPackedData, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fastlanes::bitpack_decompress::unpack_primitive_array(array: vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_fastlanes::bitpack_decompress::unpack_single(array: &vortex_fastlanes::BitPackedData, index: usize) -> vortex_array::scalar::Scalar +pub fn vortex_fastlanes::bitpack_decompress::unpack_single(array: vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>, index: usize) -> vortex_array::scalar::Scalar pub unsafe fn vortex_fastlanes::bitpack_decompress::unpack_single_primitive(packed: &[T], bit_width: usize, index_to_decode: usize) -> T @@ -78,15 +78,15 @@ pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::decode_into(&mut sel pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::initial(&mut self) -> core::option::Option<&mut [T]> -pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::new_with_strategy(strategy: S, packed: vortex_buffer::ByteBuffer, bit_width: usize, offset: usize, len: usize) -> Self - pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::trailer(&mut self) -> core::option::Option<&mut [T]> +pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::try_new_with_strategy(strategy: S, packed: vortex_buffer::ByteBuffer, bit_width: usize, offset: usize, len: usize) -> vortex_error::VortexResult + impl vortex_fastlanes::unpack_iter::UnpackedChunks pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::full_chunks(&mut self) -> vortex_fastlanes::unpack_iter::BitUnpackIterator<'_, T> -pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::new(array: &vortex_fastlanes::BitPackedData) -> Self +pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::try_new(array: &vortex_fastlanes::BitPackedData, len: usize) -> vortex_error::VortexResult pub trait vortex_fastlanes::unpack_iter::BitPacked: vortex_array::dtype::ptype::PhysicalPType @@ -124,6 +124,10 @@ pub const vortex_fastlanes::BitPacked::ID: vortex_array::array::ArrayId pub fn vortex_fastlanes::BitPacked::encode(array: &vortex_array::array::erased::ArrayRef, bit_width: u8) -> vortex_error::VortexResult +pub fn vortex_fastlanes::BitPacked::into_parts(array: vortex_fastlanes::BitPackedArray) -> vortex_fastlanes::BitPackedDataParts + +pub fn vortex_fastlanes::BitPacked::try_new(packed: vortex_array::buffer::BufferHandle, ptype: vortex_array::dtype::ptype::PType, validity: vortex_array::validity::Validity, patches: core::option::Option, bit_width: u8, len: usize, offset: u16) -> vortex_error::VortexResult + impl core::clone::Clone for vortex_fastlanes::BitPacked pub fn vortex_fastlanes::BitPacked::clone(&self) -> vortex_fastlanes::BitPacked @@ -136,8 +140,6 @@ impl vortex_array::array::vtable::VTable for vortex_fastlanes::BitPacked pub type vortex_fastlanes::BitPacked::ArrayData = vortex_fastlanes::BitPackedData -pub type vortex_fastlanes::BitPacked::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_fastlanes::BitPacked::OperationsVTable = vortex_fastlanes::BitPacked pub type vortex_fastlanes::BitPacked::ValidityVTable = vortex_fastlanes::BitPacked @@ -152,11 +154,7 @@ pub fn vortex_fastlanes::BitPacked::buffer(array: vortex_array::array::view::Arr pub fn vortex_fastlanes::BitPacked::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_fastlanes::BitPacked::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_fastlanes::BitPacked::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_fastlanes::BitPacked::dtype(array: &vortex_fastlanes::BitPackedData) -> &vortex_array::dtype::DType +pub fn vortex_fastlanes::BitPacked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_fastlanes::BitPacked::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -164,23 +162,17 @@ pub fn vortex_fastlanes::BitPacked::execute_parent(array: vortex_array::array::v pub fn vortex_fastlanes::BitPacked::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_fastlanes::BitPacked::len(array: &vortex_fastlanes::BitPackedData) -> usize - -pub fn vortex_fastlanes::BitPacked::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_fastlanes::BitPacked::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_fastlanes::BitPacked::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_fastlanes::BitPacked::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_fastlanes::BitPacked::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_fastlanes::BitPacked::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_fastlanes::BitPacked::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_fastlanes::BitPacked::stats(array: &vortex_fastlanes::BitPackedData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_fastlanes::BitPacked::vtable(_array: &vortex_fastlanes::BitPackedData) -> &Self +pub fn vortex_fastlanes::BitPacked::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_fastlanes::BitPacked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -208,35 +200,15 @@ impl vortex_array::scalar_fn::fns::cast::kernel::CastReduce for vortex_fastlanes pub fn vortex_fastlanes::BitPacked::cast(array: vortex_array::array::view::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> -pub struct vortex_fastlanes::BitPackedArrayParts - -pub vortex_fastlanes::BitPackedArrayParts::bit_width: u8 - -pub vortex_fastlanes::BitPackedArrayParts::len: usize - -pub vortex_fastlanes::BitPackedArrayParts::offset: u16 - -pub vortex_fastlanes::BitPackedArrayParts::packed: vortex_array::buffer::BufferHandle - -pub vortex_fastlanes::BitPackedArrayParts::patches: core::option::Option - -pub vortex_fastlanes::BitPackedArrayParts::validity: vortex_array::validity::Validity - pub struct vortex_fastlanes::BitPackedData impl vortex_fastlanes::BitPackedData pub fn vortex_fastlanes::BitPackedData::bit_width(&self) -> u8 -pub fn vortex_fastlanes::BitPackedData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_fastlanes::BitPackedData::encode(array: &vortex_array::array::erased::ArrayRef, bit_width: u8) -> vortex_error::VortexResult -pub fn vortex_fastlanes::BitPackedData::into_parts(self) -> vortex_fastlanes::BitPackedArrayParts - -pub fn vortex_fastlanes::BitPackedData::is_empty(&self) -> bool - -pub fn vortex_fastlanes::BitPackedData::len(&self) -> usize +pub fn vortex_fastlanes::BitPackedData::into_parts(self, len: usize, nullability: vortex_array::dtype::nullability::Nullability) -> vortex_fastlanes::BitPackedDataParts pub fn vortex_fastlanes::BitPackedData::max_packed_value(&self) -> usize @@ -246,35 +218,49 @@ pub fn vortex_fastlanes::BitPackedData::packed(&self) -> &vortex_array::buffer:: pub fn vortex_fastlanes::BitPackedData::packed_slice(&self) -> &[T] -pub fn vortex_fastlanes::BitPackedData::patches(&self) -> core::option::Option +pub fn vortex_fastlanes::BitPackedData::patch_chunk_offsets(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_fastlanes::BitPackedData::patch_indices(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_fastlanes::BitPackedData::patch_values(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_fastlanes::BitPackedData::patches(&self, len: usize) -> core::option::Option -pub fn vortex_fastlanes::BitPackedData::ptype(&self) -> vortex_array::dtype::ptype::PType +pub fn vortex_fastlanes::BitPackedData::ptype(&self, dtype: &vortex_array::dtype::DType) -> vortex_array::dtype::ptype::PType pub fn vortex_fastlanes::BitPackedData::replace_patches(&mut self, patches: core::option::Option) pub fn vortex_fastlanes::BitPackedData::try_new(packed: vortex_array::buffer::BufferHandle, ptype: vortex_array::dtype::ptype::PType, validity: vortex_array::validity::Validity, patches: core::option::Option, bit_width: u8, length: usize, offset: u16) -> vortex_error::VortexResult -pub fn vortex_fastlanes::BitPackedData::unpacked_chunks(&self) -> vortex_fastlanes::unpack_iter::BitUnpackedChunks +pub fn vortex_fastlanes::BitPackedData::unpacked_chunks(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult> -pub fn vortex_fastlanes::BitPackedData::validity(&self) -> vortex_array::validity::Validity +pub fn vortex_fastlanes::BitPackedData::validity(&self, nullability: vortex_array::dtype::nullability::Nullability) -> vortex_array::validity::Validity -pub fn vortex_fastlanes::BitPackedData::validity_mask(&self) -> vortex_mask::Mask +pub fn vortex_fastlanes::BitPackedData::validity_child(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_fastlanes::BitPackedData::validity_mask(&self, len: usize, nullability: vortex_array::dtype::nullability::Nullability) -> vortex_mask::Mask impl core::clone::Clone for vortex_fastlanes::BitPackedData pub fn vortex_fastlanes::BitPackedData::clone(&self) -> vortex_fastlanes::BitPackedData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_fastlanes::BitPackedData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_fastlanes::BitPackedData pub fn vortex_fastlanes::BitPackedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_fastlanes::BitPackedData +pub struct vortex_fastlanes::BitPackedDataParts + +pub vortex_fastlanes::BitPackedDataParts::bit_width: u8 + +pub vortex_fastlanes::BitPackedDataParts::len: usize + +pub vortex_fastlanes::BitPackedDataParts::offset: u16 + +pub vortex_fastlanes::BitPackedDataParts::packed: vortex_array::buffer::BufferHandle -pub fn vortex_fastlanes::BitPackedData::into_array(self) -> vortex_array::array::erased::ArrayRef +pub vortex_fastlanes::BitPackedDataParts::patches: core::option::Option + +pub vortex_fastlanes::BitPackedDataParts::validity: vortex_array::validity::Validity pub struct vortex_fastlanes::Delta @@ -284,6 +270,8 @@ pub const vortex_fastlanes::Delta::ID: vortex_array::array::ArrayId pub fn vortex_fastlanes::Delta::try_from_primitive_array(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fastlanes::Delta::try_new(bases: vortex_array::array::erased::ArrayRef, deltas: vortex_array::array::erased::ArrayRef, offset: usize, len: usize) -> vortex_error::VortexResult + impl core::clone::Clone for vortex_fastlanes::Delta pub fn vortex_fastlanes::Delta::clone(&self) -> vortex_fastlanes::Delta @@ -296,8 +284,6 @@ impl vortex_array::array::vtable::VTable for vortex_fastlanes::Delta pub type vortex_fastlanes::Delta::ArrayData = vortex_fastlanes::DeltaData -pub type vortex_fastlanes::Delta::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_fastlanes::Delta::OperationsVTable = vortex_fastlanes::Delta pub type vortex_fastlanes::Delta::ValidityVTable = vortex_fastlanes::Delta @@ -310,33 +296,23 @@ pub fn vortex_fastlanes::Delta::buffer(_array: vortex_array::array::view::ArrayV pub fn vortex_fastlanes::Delta::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_fastlanes::Delta::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_fastlanes::Delta::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_fastlanes::Delta::dtype(array: &vortex_fastlanes::DeltaData) -> &vortex_array::dtype::DType +pub fn vortex_fastlanes::Delta::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_fastlanes::Delta::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_fastlanes::Delta::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_fastlanes::Delta::len(array: &vortex_fastlanes::DeltaData) -> usize - -pub fn vortex_fastlanes::Delta::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_fastlanes::Delta::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_fastlanes::Delta::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_fastlanes::Delta::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_fastlanes::Delta::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_fastlanes::Delta::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_fastlanes::Delta::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_fastlanes::Delta::stats(array: &vortex_fastlanes::DeltaData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_fastlanes::Delta::vtable(_array: &vortex_fastlanes::DeltaData) -> &Self +pub fn vortex_fastlanes::Delta::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_fastlanes::Delta::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -364,34 +340,18 @@ pub fn vortex_fastlanes::DeltaData::bases(&self) -> &vortex_array::array::erased pub fn vortex_fastlanes::DeltaData::deltas(&self) -> &vortex_array::array::erased::ArrayRef -pub fn vortex_fastlanes::DeltaData::dtype(&self) -> &vortex_array::dtype::DType - -pub fn vortex_fastlanes::DeltaData::is_empty(&self) -> bool - -pub fn vortex_fastlanes::DeltaData::len(&self) -> usize - pub fn vortex_fastlanes::DeltaData::offset(&self) -> usize -pub fn vortex_fastlanes::DeltaData::try_from_primitive_array(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult - pub fn vortex_fastlanes::DeltaData::try_new(bases: vortex_array::array::erased::ArrayRef, deltas: vortex_array::array::erased::ArrayRef, offset: usize, len: usize) -> vortex_error::VortexResult impl core::clone::Clone for vortex_fastlanes::DeltaData pub fn vortex_fastlanes::DeltaData::clone(&self) -> vortex_fastlanes::DeltaData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_fastlanes::DeltaData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_fastlanes::DeltaData pub fn vortex_fastlanes::DeltaData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_fastlanes::DeltaData - -pub fn vortex_fastlanes::DeltaData::into_array(self) -> vortex_array::array::erased::ArrayRef - pub struct vortex_fastlanes::FoR impl vortex_fastlanes::FoR @@ -414,8 +374,6 @@ impl vortex_array::array::vtable::VTable for vortex_fastlanes::FoR pub type vortex_fastlanes::FoR::ArrayData = vortex_fastlanes::FoRData -pub type vortex_fastlanes::FoR::Metadata = vortex_array::scalar::Scalar - pub type vortex_fastlanes::FoR::OperationsVTable = vortex_fastlanes::FoR pub type vortex_fastlanes::FoR::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -428,11 +386,7 @@ pub fn vortex_fastlanes::FoR::buffer(_array: vortex_array::array::view::ArrayVie pub fn vortex_fastlanes::FoR::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_fastlanes::FoR::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_fastlanes::FoR::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_fastlanes::FoR::dtype(array: &vortex_fastlanes::FoRData) -> &vortex_array::dtype::DType +pub fn vortex_fastlanes::FoR::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_fastlanes::FoR::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -440,23 +394,17 @@ pub fn vortex_fastlanes::FoR::execute_parent(array: vortex_array::array::view::A pub fn vortex_fastlanes::FoR::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_fastlanes::FoR::len(array: &vortex_fastlanes::FoRData) -> usize - -pub fn vortex_fastlanes::FoR::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_fastlanes::FoR::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_fastlanes::FoR::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_fastlanes::FoR::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_fastlanes::FoR::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_fastlanes::FoR::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_fastlanes::FoR::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_fastlanes::FoR::stats(array: &vortex_fastlanes::FoRData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_fastlanes::FoR::vtable(_array: &vortex_fastlanes::FoRData) -> &Self +pub fn vortex_fastlanes::FoR::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_fastlanes::FoR::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -504,8 +452,6 @@ pub fn vortex_fastlanes::FoRData::ptype(&self) -> vortex_array::dtype::ptype::PT pub fn vortex_fastlanes::FoRData::reference_scalar(&self) -> &vortex_array::scalar::Scalar -pub fn vortex_fastlanes::FoRData::try_new(encoded: vortex_array::array::erased::ArrayRef, reference: vortex_array::scalar::Scalar) -> vortex_error::VortexResult - impl vortex_fastlanes::FoRData pub fn vortex_fastlanes::FoRData::encode(array: vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult @@ -514,18 +460,10 @@ impl core::clone::Clone for vortex_fastlanes::FoRData pub fn vortex_fastlanes::FoRData::clone(&self) -> vortex_fastlanes::FoRData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_fastlanes::FoRData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_fastlanes::FoRData pub fn vortex_fastlanes::FoRData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_fastlanes::FoRData - -pub fn vortex_fastlanes::FoRData::into_array(self) -> vortex_array::array::erased::ArrayRef - pub struct vortex_fastlanes::RLE impl vortex_fastlanes::RLE @@ -534,7 +472,9 @@ pub const vortex_fastlanes::RLE::ID: vortex_array::array::ArrayId pub fn vortex_fastlanes::RLE::encode(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult -pub unsafe fn vortex_fastlanes::RLE::new_unchecked(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, dtype: vortex_array::dtype::DType, offset: usize, length: usize) -> vortex_fastlanes::RLEArray +pub unsafe fn vortex_fastlanes::RLE::new_unchecked(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> vortex_fastlanes::RLEArray + +pub fn vortex_fastlanes::RLE::try_new(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> vortex_error::VortexResult impl core::clone::Clone for vortex_fastlanes::RLE @@ -548,11 +488,9 @@ impl vortex_array::array::vtable::VTable for vortex_fastlanes::RLE pub type vortex_fastlanes::RLE::ArrayData = vortex_fastlanes::RLEData -pub type vortex_fastlanes::RLE::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_fastlanes::RLE::OperationsVTable = vortex_fastlanes::RLE -pub type vortex_fastlanes::RLE::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChildSliceHelper +pub type vortex_fastlanes::RLE::ValidityVTable = vortex_fastlanes::RLE pub fn vortex_fastlanes::RLE::array_eq(array: &vortex_fastlanes::RLEData, other: &vortex_fastlanes::RLEData, precision: vortex_array::hash::Precision) -> bool @@ -562,11 +500,7 @@ pub fn vortex_fastlanes::RLE::buffer(_array: vortex_array::array::view::ArrayVie pub fn vortex_fastlanes::RLE::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_fastlanes::RLE::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_fastlanes::RLE::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_fastlanes::RLE::dtype(array: &vortex_fastlanes::RLEData) -> &vortex_array::dtype::DType +pub fn vortex_fastlanes::RLE::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_fastlanes::RLE::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -574,23 +508,17 @@ pub fn vortex_fastlanes::RLE::execute_parent(array: vortex_array::array::view::A pub fn vortex_fastlanes::RLE::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_fastlanes::RLE::len(array: &vortex_fastlanes::RLEData) -> usize - -pub fn vortex_fastlanes::RLE::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_fastlanes::RLE::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_fastlanes::RLE::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_fastlanes::RLE::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_fastlanes::RLE::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_fastlanes::RLE::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_fastlanes::RLE::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_fastlanes::RLE::stats(array: &vortex_fastlanes::RLEData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_fastlanes::RLE::vtable(_array: &vortex_fastlanes::RLEData) -> &Self +pub fn vortex_fastlanes::RLE::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_fastlanes::RLE::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -598,9 +526,9 @@ impl vortex_array::array::vtable::operations::OperationsVTable, index: usize, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -impl vortex_array::array::vtable::validity::ValidityChild for vortex_fastlanes::RLE +impl vortex_array::array::vtable::validity::ValidityVTable for vortex_fastlanes::RLE -pub fn vortex_fastlanes::RLE::validity_child(array: &vortex_fastlanes::RLEData) -> &vortex_array::array::erased::ArrayRef +pub fn vortex_fastlanes::RLE::validity(array: vortex_array::array::view::ArrayView<'_, vortex_fastlanes::RLE>) -> vortex_error::VortexResult impl vortex_array::arrays::slice::SliceKernel for vortex_fastlanes::RLE @@ -614,15 +542,13 @@ pub struct vortex_fastlanes::RLEData impl vortex_fastlanes::RLEData -pub fn vortex_fastlanes::RLEData::dtype(&self) -> &vortex_array::dtype::DType - -pub fn vortex_fastlanes::RLEData::indices(&self) -> &vortex_array::array::erased::ArrayRef +pub fn vortex_fastlanes::RLEData::encode(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult -pub fn vortex_fastlanes::RLEData::is_empty(&self) -> bool +impl vortex_fastlanes::RLEData -pub fn vortex_fastlanes::RLEData::len(&self) -> usize +pub fn vortex_fastlanes::RLEData::indices(&self) -> &vortex_array::array::erased::ArrayRef -pub unsafe fn vortex_fastlanes::RLEData::new_unchecked(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, dtype: vortex_array::dtype::DType, offset: usize, length: usize) -> Self +pub unsafe fn vortex_fastlanes::RLEData::new_unchecked(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, offset: usize) -> Self pub fn vortex_fastlanes::RLEData::offset(&self) -> usize @@ -632,29 +558,109 @@ pub fn vortex_fastlanes::RLEData::values(&self) -> &vortex_array::array::erased: pub fn vortex_fastlanes::RLEData::values_idx_offsets(&self) -> &vortex_array::array::erased::ArrayRef -impl vortex_fastlanes::RLEData - -pub fn vortex_fastlanes::RLEData::encode(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult - impl core::clone::Clone for vortex_fastlanes::RLEData pub fn vortex_fastlanes::RLEData::clone(&self) -> vortex_fastlanes::RLEData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_fastlanes::RLEData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_fastlanes::RLEData pub fn vortex_fastlanes::RLEData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_fastlanes::RLEData +pub trait vortex_fastlanes::BitPackedArrayExt + +pub fn vortex_fastlanes::BitPackedArrayExt::bit_width(&self) -> u8 + +pub fn vortex_fastlanes::BitPackedArrayExt::bitpacked_data(&self) -> &vortex_fastlanes::BitPackedData + +pub fn vortex_fastlanes::BitPackedArrayExt::bitpacked_dtype(&self) -> &vortex_array::dtype::DType + +pub fn vortex_fastlanes::BitPackedArrayExt::bitpacked_len(&self) -> usize + +pub fn vortex_fastlanes::BitPackedArrayExt::offset(&self) -> u16 + +pub fn vortex_fastlanes::BitPackedArrayExt::packed(&self) -> &vortex_array::buffer::BufferHandle + +pub fn vortex_fastlanes::BitPackedArrayExt::packed_slice(&self) -> &[T] + +pub fn vortex_fastlanes::BitPackedArrayExt::patch_chunk_offsets(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_fastlanes::BitPackedArrayExt::patch_indices(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_fastlanes::BitPackedArrayExt::patch_values(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_fastlanes::BitPackedArrayExt::patches(&self) -> core::option::Option + +pub fn vortex_fastlanes::BitPackedArrayExt::unpacked_chunks(&self) -> vortex_error::VortexResult> + +pub fn vortex_fastlanes::BitPackedArrayExt::validity(&self) -> vortex_array::validity::Validity + +pub fn vortex_fastlanes::BitPackedArrayExt::validity_child(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_fastlanes::BitPackedArrayExt::validity_mask(&self) -> vortex_mask::Mask + +impl vortex_fastlanes::BitPackedArrayExt for vortex_array::array::typed::Array + +pub fn vortex_array::array::typed::Array::bit_width(&self) -> u8 + +pub fn vortex_array::array::typed::Array::bitpacked_data(&self) -> &vortex_fastlanes::BitPackedData + +pub fn vortex_array::array::typed::Array::bitpacked_dtype(&self) -> &vortex_array::dtype::DType + +pub fn vortex_array::array::typed::Array::bitpacked_len(&self) -> usize + +pub fn vortex_array::array::typed::Array::offset(&self) -> u16 + +pub fn vortex_array::array::typed::Array::packed(&self) -> &vortex_array::buffer::BufferHandle + +pub fn vortex_array::array::typed::Array::packed_slice(&self) -> &[T] + +pub fn vortex_array::array::typed::Array::patch_chunk_offsets(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_array::array::typed::Array::patch_indices(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_array::array::typed::Array::patch_values(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_array::array::typed::Array::patches(&self) -> core::option::Option + +pub fn vortex_array::array::typed::Array::unpacked_chunks(&self) -> vortex_error::VortexResult> + +pub fn vortex_array::array::typed::Array::validity(&self) -> vortex_array::validity::Validity + +pub fn vortex_array::array::typed::Array::validity_child(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_array::array::typed::Array::validity_mask(&self) -> vortex_mask::Mask + +impl vortex_fastlanes::BitPackedArrayExt for vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked> + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::bit_width(&self) -> u8 + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::bitpacked_data(&self) -> &vortex_fastlanes::BitPackedData + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::bitpacked_dtype(&self) -> &vortex_array::dtype::DType + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::bitpacked_len(&self) -> usize + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::offset(&self) -> u16 + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::packed(&self) -> &vortex_array::buffer::BufferHandle + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::packed_slice(&self) -> &[T] + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::patch_chunk_offsets(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::patch_indices(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::patch_values(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::patches(&self) -> core::option::Option + +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::unpacked_chunks(&self) -> vortex_error::VortexResult> -pub fn vortex_fastlanes::RLEData::into_array(self) -> vortex_array::array::erased::ArrayRef +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::validity(&self) -> vortex_array::validity::Validity -impl vortex_array::array::vtable::validity::ValidityChildSliceHelper for vortex_fastlanes::RLEData +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::validity_child(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> -pub fn vortex_fastlanes::RLEData::unsliced_child_and_slice(&self) -> (&vortex_array::array::erased::ArrayRef, usize, usize) +pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::validity_mask(&self) -> vortex_mask::Mask pub fn vortex_fastlanes::delta_compress(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<(vortex_array::arrays::primitive::vtable::PrimitiveArray, vortex_array::arrays::primitive::vtable::PrimitiveArray)> diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index 43ae4e410c5..4290e5cb44a 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -23,8 +23,8 @@ use vortex_error::vortex_bail; use vortex_mask::AllOr; use vortex_mask::Mask; +use crate::BitPacked; use crate::BitPackedArray; -use crate::BitPackedData; use crate::bitpack_decompress; pub fn bitpack_to_best_bit_width(array: &PrimitiveArray) -> VortexResult { @@ -71,27 +71,16 @@ pub fn bitpack_encode( .transpose()? .flatten(); - // SAFETY: all components validated above - let bitpacked = unsafe { - BitPackedData::new_unchecked( - BufferHandle::new_host(packed), - array.dtype().clone(), - array.validity(), - patches, - bit_width, - array.len(), - 0, - ) - }; - let bitpacked = - BitPackedArray::try_from_data(bitpacked).vortex_expect("BitPackedData is always valid"); - { - let bp_ref = bitpacked.clone().into_array(); - bitpacked - .stats_set - .to_ref(&bp_ref) - .inherit_from(array.statistics()); - } + let bitpacked = BitPacked::try_new( + BufferHandle::new_host(packed), + array.ptype(), + array.validity(), + patches, + bit_width, + array.len(), + 0, + )?; + bitpacked.statistics().inherit_from(array.statistics()); Ok(bitpacked) } @@ -110,28 +99,18 @@ pub unsafe fn bitpack_encode_unchecked( // SAFETY: non-negativity of input checked by caller. let packed = unsafe { bitpack_unchecked(&array, bit_width) }; - // SAFETY: checked by bitpack_unchecked - let data = unsafe { - BitPackedData::new_unchecked( - BufferHandle::new_host(packed), - array.dtype().clone(), - array.validity(), - None, - bit_width, - array.len(), - 0, - ) - }; - let bitpacked = - BitPackedArray::try_from_data(data).vortex_expect("BitPackedData is always valid"); - { - let bp_ref = bitpacked.clone().into_array(); - let arr_ref = array.into_array(); - bitpacked - .stats_set - .to_ref(&bp_ref) - .inherit_from(arr_ref.statistics()); - } + let arr_ref = array.clone().into_array(); + let bitpacked = BitPacked::try_new( + BufferHandle::new_host(packed), + array.ptype(), + array.validity(), + None, + bit_width, + array.len(), + 0, + ) + .vortex_expect("bitpacked array construction should succeed"); + bitpacked.statistics().inherit_from(arr_ref.statistics()); Ok(bitpacked) } @@ -444,6 +423,7 @@ mod test { use vortex_session::VortexSession; use super::*; + use crate::BitPackedData; use crate::bitpack_compress::test_harness::make_array; static SESSION: LazyLock = @@ -469,7 +449,7 @@ mod test { ); assert!(values.ptype().is_unsigned_int()); let compressed = BitPackedData::encode(&values.into_array(), 4).unwrap(); - assert!(compressed.patches().is_none()); + assert!(compressed.patches(compressed.len()).is_none()); assert_eq!( (0..(1 << 4)).collect::>(), compressed @@ -531,7 +511,7 @@ mod test { let array = PrimitiveArray::from_iter(values); let bitpacked = bitpack_encode(&array, 4, None).unwrap(); - let patches = bitpacked.patches().unwrap(); + let patches = bitpacked.patches(bitpacked.len()).unwrap(); let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); // chunk 0 (0-1023): patches at 100, 200 -> starts at patch index 0 @@ -554,7 +534,7 @@ mod test { let array = PrimitiveArray::from_iter(values); let bitpacked = bitpack_encode(&array, 4, None).unwrap(); - let patches = bitpacked.patches().unwrap(); + let patches = bitpacked.patches(bitpacked.len()).unwrap(); let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); assert_arrays_eq!(chunk_offsets, PrimitiveArray::from_iter([0u64, 2, 2])); @@ -573,7 +553,7 @@ mod test { let array = PrimitiveArray::from_iter(values); let bitpacked = bitpack_encode(&array, 4, None).unwrap(); - let patches = bitpacked.patches().unwrap(); + let patches = bitpacked.patches(bitpacked.len()).unwrap(); let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); // chunk 0 (0-1023): patches at 100, 200 -> starts at patch index 0 @@ -597,7 +577,7 @@ mod test { let array = PrimitiveArray::from_iter(values); let bitpacked = bitpack_encode(&array, 4, None).unwrap(); - let patches = bitpacked.patches().unwrap(); + let patches = bitpacked.patches(bitpacked.len()).unwrap(); let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); // Single chunk starting at patch index 0. diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs index 4f31154d598..5ed8d6b89fa 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs @@ -4,6 +4,7 @@ use fastlanes::BitPacking; use itertools::Itertools; use num_traits::AsPrimitive; +use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::arrays::PrimitiveArray; use vortex_array::builders::ArrayBuilder; @@ -17,18 +18,22 @@ use vortex_array::scalar::Scalar; use vortex_error::VortexExpect; use vortex_error::VortexResult; -use crate::BitPackedData; -use crate::unpack_iter::BitPacked; +use crate::BitPacked; +use crate::BitPackedArrayExt; +use crate::unpack_iter::BitPacked as BitPackedUnpack; /// Unpacks a bit-packed array into a primitive array. -pub fn unpack_array(array: &BitPackedData, ctx: &mut ExecutionCtx) -> VortexResult { - match_each_integer_ptype!(array.ptype(), |P| { +pub fn unpack_array( + array: ArrayView<'_, BitPacked>, + ctx: &mut ExecutionCtx, +) -> VortexResult { + match_each_integer_ptype!(array.dtype().as_ptype(), |P| { unpack_primitive_array::

(array, ctx) }) } -pub fn unpack_primitive_array( - array: &BitPackedData, +pub fn unpack_primitive_array( + array: ArrayView<'_, BitPacked>, ctx: &mut ExecutionCtx, ) -> VortexResult { let mut builder = PrimitiveBuilder::with_capacity(array.dtype().nullability(), array.len()); @@ -37,8 +42,8 @@ pub fn unpack_primitive_array( Ok(builder.finish_into_primitive()) } -pub(crate) fn unpack_into_primitive_builder( - array: &BitPackedData, +pub(crate) fn unpack_into_primitive_builder( + array: ArrayView<'_, BitPacked>, // TODO(ngates): do we want to use fastlanes alignment for this buffer? builder: &mut PrimitiveBuilder, ctx: &mut ExecutionCtx, @@ -59,7 +64,7 @@ pub(crate) fn unpack_into_primitive_builder( // SAFETY: `decode_into` will initialize all values in this range. let uninit_slice = unsafe { uninit_range.slice_uninit_mut(0, array.len()) }; - let mut bit_packed_iter = array.unpacked_chunks(); + let mut bit_packed_iter = array.unpacked_chunks()?; bit_packed_iter.decode_into(uninit_slice); if let Some(ref patches) = array.patches() { @@ -106,9 +111,9 @@ pub fn apply_patches_to_uninit_range_fn T>( Ok(()) } -pub fn unpack_single(array: &BitPackedData, index: usize) -> Scalar { +pub fn unpack_single(array: ArrayView<'_, BitPacked>, index: usize) -> Scalar { let bit_width = array.bit_width() as usize; - let ptype = array.ptype(); + let ptype = array.dtype().as_ptype(); // let packed = array.packed().into_primitive()?; let index_in_encoded = index + array.offset() as usize; let scalar: Scalar = match_each_unsigned_integer_ptype!(ptype.to_unsigned(), |P| { @@ -178,6 +183,10 @@ mod tests { static SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); + fn unpack(bitpacked: &BitPackedArray) -> VortexResult { + unpack_array(bitpacked.as_view(), &mut SESSION.create_execution_ctx()) + } + fn compression_roundtrip(n: usize) { let values = PrimitiveArray::from_iter((0..n).map(|i| (i % 2047) as u16)); let compressed = BitPackedData::encode(&values.clone().into_array(), 11).unwrap(); @@ -188,7 +197,9 @@ mod tests { .iter() .enumerate() .for_each(|(i, v)| { - let scalar: u16 = (&unpack_single(&compressed, i)).try_into().unwrap(); + let scalar: u16 = (&unpack_single(compressed.as_view(), i)) + .try_into() + .unwrap(); assert_eq!(scalar, *v); }); } @@ -210,7 +221,7 @@ mod tests { fn test_all_zeros() -> VortexResult<()> { let zeros = buffer![0u16, 0, 0, 0].into_array().to_primitive(); let bitpacked = encode(&zeros, 0); - let actual = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; + let actual = unpack(&bitpacked)?; assert_arrays_eq!(actual, PrimitiveArray::from_iter([0u16, 0, 0, 0])); Ok(()) } @@ -219,7 +230,7 @@ mod tests { fn test_simple_patches() -> VortexResult<()> { let zeros = buffer![0u16, 1, 0, 1].into_array().to_primitive(); let bitpacked = encode(&zeros, 0); - let actual = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; + let actual = unpack(&bitpacked)?; assert_arrays_eq!(actual, PrimitiveArray::from_iter([0u16, 1, 0, 1])); Ok(()) } @@ -228,7 +239,7 @@ mod tests { fn test_one_full_chunk() -> VortexResult<()> { let zeros = BufferMut::from_iter(0u16..1024).into_array().to_primitive(); let bitpacked = encode(&zeros, 10); - let actual = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; + let actual = unpack(&bitpacked)?; assert_arrays_eq!(actual, PrimitiveArray::from_iter(0u16..1024)); Ok(()) } @@ -240,7 +251,7 @@ mod tests { .to_primitive(); let bitpacked = encode(&zeros, 10); assert!(bitpacked.patches().is_some()); - let actual = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; + let actual = unpack(&bitpacked)?; assert_arrays_eq!( actual, PrimitiveArray::from_iter((5u16..1029).chain(5u16..1029).chain(5u16..1029)) @@ -253,7 +264,7 @@ mod tests { let zeros = BufferMut::from_iter(0u16..1025).into_array().to_primitive(); let bitpacked = encode(&zeros, 11); assert!(bitpacked.patches().is_none()); - let actual = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; + let actual = unpack(&bitpacked)?; assert_arrays_eq!(actual, PrimitiveArray::from_iter(0u16..1025)); Ok(()) } @@ -266,7 +277,7 @@ mod tests { let bitpacked = encode(&zeros, 10); assert_eq!(bitpacked.len(), 1025); assert!(bitpacked.patches().is_some()); - let actual = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; + let actual = unpack(&bitpacked)?; assert_arrays_eq!(actual, PrimitiveArray::from_iter(512u16..1537)); Ok(()) } @@ -321,7 +332,7 @@ mod tests { let mut builder = PrimitiveBuilder::::new(Nullability::NonNullable); unpack_into_primitive_builder( - &bitpacked, + bitpacked.as_view(), &mut builder, &mut SESSION.create_execution_ctx(), )?; @@ -349,7 +360,7 @@ mod tests { // Unpack into a new builder. let mut builder = PrimitiveBuilder::::with_capacity(Nullability::Nullable, 5); unpack_into_primitive_builder( - &bitpacked, + bitpacked.as_view(), &mut builder, &mut SESSION.create_execution_ctx(), )?; @@ -385,7 +396,7 @@ mod tests { // Unpack into a new builder. let mut builder = PrimitiveBuilder::::with_capacity(Nullability::NonNullable, 100); unpack_into_primitive_builder( - &bitpacked, + bitpacked.as_view(), &mut builder, &mut SESSION.create_execution_ctx(), )?; @@ -426,7 +437,7 @@ mod tests { let large_bitpacked = encode(&large_array, 8); assert!(large_bitpacked.patches().is_some()); - let large_result = unpack_array(&large_bitpacked, &mut SESSION.create_execution_ctx())?; + let large_result = unpack(&large_bitpacked)?; assert_eq!(large_result.len(), 3072); Ok(()) } @@ -440,8 +451,7 @@ mod tests { let array = PrimitiveArray::new(values, validity); let bitpacked = encode(&array, 9); - let result = - unpack_array(&bitpacked, &mut SESSION.create_execution_ctx()).vortex_expect("unpack"); + let result = unpack(&bitpacked).vortex_expect("unpack"); // Verify length. assert_eq!(result.len(), 7); @@ -458,8 +468,7 @@ mod tests { let patch_bitpacked = encode(&patch_array, 5); assert!(patch_bitpacked.patches().is_some()); - let patch_result = unpack_array(&patch_bitpacked, &mut SESSION.create_execution_ctx()) - .vortex_expect("unpack"); + let patch_result = unpack(&patch_bitpacked).vortex_expect("unpack"); assert_eq!(patch_result.len(), 7); // Test all nulls edge case. @@ -468,8 +477,7 @@ mod tests { Validity::from_iter([false, false, false, false]), ); let all_nulls_bp = encode(&all_nulls, 0); - let all_nulls_result = unpack_array(&all_nulls_bp, &mut SESSION.create_execution_ctx()) - .vortex_expect("unpack"); + let all_nulls_result = unpack(&all_nulls_bp).vortex_expect("unpack"); assert_eq!(all_nulls_result.len(), 4); } @@ -480,7 +488,7 @@ mod tests { let test_consistency = |array: &PrimitiveArray, bit_width: u8| -> VortexResult<()> { let bitpacked = encode(array, bit_width); - let unpacked_array = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; + let unpacked_array = unpack(&bitpacked)?; let executed = { let mut ctx = SESSION.create_execution_ctx(); @@ -584,23 +592,23 @@ mod tests { // Empty array. let empty: PrimitiveArray = PrimitiveArray::from_iter(Vec::::new()); let empty_bp = encode(&empty, 0); - let empty_result = unpack_array(&empty_bp, &mut SESSION.create_execution_ctx())?; + let empty_result = unpack(&empty_bp)?; assert_eq!(empty_result.len(), 0); // All zeros (bit_width = 0). let zeros = PrimitiveArray::from_iter([0u32; 100]); let zeros_bp = encode(&zeros, 0); - let zeros_result = unpack_array(&zeros_bp, &mut SESSION.create_execution_ctx())?; + let zeros_result = unpack(&zeros_bp)?; assert_eq!(zeros_result.len(), 100); // Verify consistency with unpack_array. - let zeros_array = unpack_array(&zeros_bp, &mut SESSION.create_execution_ctx())?; + let zeros_array = unpack(&zeros_bp)?; assert_eq!(zeros_result.len(), zeros_array.len()); assert_arrays_eq!(zeros_result, zeros_array); // Maximum bit width for u16 (15 bits, since bitpacking requires bit_width < type bit width). let max_values = PrimitiveArray::from_iter([32767u16; 50]); // 2^15 - 1 let max_bp = encode(&max_values, 15); - let max_result = unpack_array(&max_bp, &mut SESSION.create_execution_ctx())?; + let max_result = unpack(&max_bp)?; assert_eq!(max_result.len(), 50); // Exactly 3072 elements with patches across chunks. @@ -617,17 +625,17 @@ mod tests { let boundary_bp = encode(&boundary_array, 7); assert!(boundary_bp.patches().is_some()); - let boundary_result = unpack_array(&boundary_bp, &mut SESSION.create_execution_ctx())?; + let boundary_result = unpack(&boundary_bp)?; assert_eq!(boundary_result.len(), 3072); // Verify consistency. - let boundary_unpacked = unpack_array(&boundary_bp, &mut SESSION.create_execution_ctx())?; + let boundary_unpacked = unpack(&boundary_bp)?; assert_eq!(boundary_result.len(), boundary_unpacked.len()); assert_arrays_eq!(boundary_result, boundary_unpacked); // Single element. let single = PrimitiveArray::from_iter([42u8]); let single_bp = encode(&single, 6); - let single_result = unpack_array(&single_bp, &mut SESSION.create_execution_ctx())?; + let single_result = unpack(&single_bp)?; assert_eq!(single_result.len(), 1); Ok(()) } diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index f54e788650b..9e8c98b1c04 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -2,15 +2,17 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use fastlanes::BitPacking; +use vortex_array::Array; use vortex_array::ArrayRef; +use vortex_array::ArrayView; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; +use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::patches::Patches; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable::child_to_validity; use vortex_array::vtable::validity_to_child; @@ -44,7 +46,7 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = [ "validity", ]; -pub struct BitPackedArrayParts { +pub struct BitPackedDataParts { pub offset: u16, pub bit_width: u8, pub len: usize, @@ -59,15 +61,12 @@ pub struct BitPackedData { /// The offset within the first block (created with a slice). /// 0 <= offset < 1024 pub(super) offset: u16, - pub(super) len: usize, - pub(super) dtype: DType, pub(super) bit_width: u8, pub(super) packed: BufferHandle, /// The offset metadata from patches, needed to reconstruct Patches from slots. pub(super) patch_offset: Option, /// The offset_within_chunk metadata from patches. pub(super) patch_offset_within_chunk: Option, - pub(super) stats_set: ArrayStats, } impl BitPackedData { @@ -93,7 +92,6 @@ impl BitPackedData { /// bit-packed array. pub(crate) unsafe fn new_unchecked( packed: BufferHandle, - dtype: DType, validity: Validity, patches: Option, bit_width: u8, @@ -109,13 +107,10 @@ impl BitPackedData { Self { slots, offset, - len, - dtype, bit_width, packed, patch_offset, patch_offset_within_chunk, - stats_set: Default::default(), } } @@ -176,16 +171,28 @@ impl BitPackedData { offset, )?; - let dtype = DType::Primitive(ptype, validity.nullability()); - // SAFETY: all components validated above unsafe { Ok(Self::new_unchecked( - packed, dtype, validity, patches, bit_width, length, offset, + packed, validity, patches, bit_width, length, offset, )) } } + pub(crate) fn validate_against_outer(&self, dtype: &DType, len: usize) -> VortexResult<()> { + let validity = self.validity(dtype.nullability()); + let patches = self.patches(len); + Self::validate( + &self.packed, + dtype.as_ptype(), + &validity, + patches.as_ref(), + self.bit_width, + len, + self.offset, + ) + } + fn validate( packed: &BufferHandle, ptype: PType, @@ -247,31 +254,13 @@ impl BitPackedData { Ok(()) } - /// Returns the length of the array. - #[inline] - pub fn len(&self) -> usize { - self.len - } - - /// Returns `true` if the array is empty. - #[inline] - pub fn is_empty(&self) -> bool { - self.len == 0 - } - - /// Returns the dtype of the array. - #[inline] - pub fn dtype(&self) -> &DType { - &self.dtype - } - /// Returns the validity as a [`Mask`](vortex_mask::Mask). - pub fn validity_mask(&self) -> vortex_mask::Mask { - self.validity().to_mask(self.len()) + pub fn validity_mask(&self, len: usize, nullability: Nullability) -> vortex_mask::Mask { + self.validity(nullability).to_mask(len) } - pub fn ptype(&self) -> PType { - self.dtype.as_ptype() + pub fn ptype(&self, dtype: &DType) -> PType { + dtype.as_ptype() } /// Underlying bit packed values as byte array @@ -294,14 +283,38 @@ impl BitPackedData { unsafe { std::slice::from_raw_parts(packed_ptr, packed_len) } } + #[inline] + pub fn patch_indices(&self) -> Option<&ArrayRef> { + self.slots[PATCH_INDICES_SLOT].as_ref() + } + + #[inline] + pub fn patch_values(&self) -> Option<&ArrayRef> { + self.slots[PATCH_VALUES_SLOT].as_ref() + } + + #[inline] + pub fn patch_chunk_offsets(&self) -> Option<&ArrayRef> { + self.slots[PATCH_CHUNK_OFFSETS_SLOT].as_ref() + } + + #[inline] + pub fn validity_child(&self) -> Option<&ArrayRef> { + self.slots[VALIDITY_SLOT].as_ref() + } + /// Accessor for bit unpacked chunks - pub fn unpacked_chunks(&self) -> BitUnpackedChunks { + pub fn unpacked_chunks( + &self, + dtype: &DType, + len: usize, + ) -> VortexResult> { assert_eq!( T::PTYPE, - self.ptype(), + self.ptype(dtype), "Requested type doesn't match the array ptype" ); - BitUnpackedChunks::new(self) + BitUnpackedChunks::try_new(self, len) } /// Bit-width of the packed values @@ -315,22 +328,19 @@ impl BitPackedData { /// Reconstructs a `Patches` from the stored slots and patch metadata. /// If present, patches MUST be a `SparseArray` with equal-length to this array, and whose /// indices indicate the locations of patches. The indices must have non-zero length. - pub fn patches(&self) -> Option { - match ( - &self.slots[PATCH_INDICES_SLOT], - &self.slots[PATCH_VALUES_SLOT], - ) { + pub fn patches(&self, len: usize) -> Option { + match (self.patch_indices(), self.patch_values()) { (Some(indices), Some(values)) => { let patch_offset = self .patch_offset .vortex_expect("has patch slots but no patch_offset"); Some(unsafe { Patches::new_unchecked( - self.len, + len, patch_offset, indices.clone(), values.clone(), - self.slots[PATCH_CHUNK_OFFSETS_SLOT].clone(), + self.patch_chunk_offsets().cloned(), self.patch_offset_within_chunk, ) }) @@ -340,8 +350,8 @@ impl BitPackedData { } /// Returns the validity, reconstructed from the stored slot. - pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + pub fn validity(&self, nullability: Nullability) -> Validity { + child_to_validity(&self.validity_child().cloned(), nullability) } pub fn replace_patches(&mut self, patches: Option) { @@ -392,13 +402,13 @@ impl BitPackedData { (1 << self.bit_width()) - 1 } - pub fn into_parts(self) -> BitPackedArrayParts { - let patches = self.patches(); - let validity = self.validity(); - BitPackedArrayParts { + pub fn into_parts(self, len: usize, nullability: Nullability) -> BitPackedDataParts { + let patches = self.patches(len); + let validity = self.validity(nullability); + BitPackedDataParts { offset: self.offset, bit_width: self.bit_width, - len: self.len, + len, packed: self.packed, patches, validity, @@ -406,6 +416,102 @@ impl BitPackedData { } } +pub trait BitPackedArrayExt { + fn bitpacked_data(&self) -> &BitPackedData; + fn bitpacked_dtype(&self) -> &DType; + fn bitpacked_len(&self) -> usize; + + #[inline] + fn packed(&self) -> &BufferHandle { + self.bitpacked_data().packed() + } + + #[inline] + fn bit_width(&self) -> u8 { + self.bitpacked_data().bit_width() + } + + #[inline] + fn offset(&self) -> u16 { + self.bitpacked_data().offset() + } + + #[inline] + fn patch_indices(&self) -> Option<&ArrayRef> { + self.bitpacked_data().patch_indices() + } + + #[inline] + fn patch_values(&self) -> Option<&ArrayRef> { + self.bitpacked_data().patch_values() + } + + #[inline] + fn patch_chunk_offsets(&self) -> Option<&ArrayRef> { + self.bitpacked_data().patch_chunk_offsets() + } + + #[inline] + fn validity_child(&self) -> Option<&ArrayRef> { + self.bitpacked_data().validity_child() + } + + #[inline] + fn patches(&self) -> Option { + self.bitpacked_data().patches(self.bitpacked_len()) + } + + #[inline] + fn validity(&self) -> Validity { + self.bitpacked_data() + .validity(self.bitpacked_dtype().nullability()) + } + + #[inline] + fn validity_mask(&self) -> vortex_mask::Mask { + self.validity().to_mask(self.bitpacked_len()) + } + + #[inline] + fn packed_slice(&self) -> &[T] { + self.bitpacked_data().packed_slice::() + } + + #[inline] + fn unpacked_chunks(&self) -> VortexResult> { + self.bitpacked_data() + .unpacked_chunks::(self.bitpacked_dtype(), self.bitpacked_len()) + } +} + +impl BitPackedArrayExt for Array { + fn bitpacked_data(&self) -> &BitPackedData { + self.data() + } + + fn bitpacked_dtype(&self) -> &DType { + self.dtype() + } + + fn bitpacked_len(&self) -> usize { + self.len() + } +} + +impl BitPackedArrayExt for ArrayView<'_, crate::BitPacked> { + fn bitpacked_data(&self) -> &BitPackedData { + self.data() + } + + fn bitpacked_dtype(&self) -> &DType { + self.dtype() + } + + fn bitpacked_len(&self) -> usize { + self.len() + } +} + #[cfg(test)] mod test { use vortex_array::IntoArray; @@ -449,7 +555,11 @@ mod test { let parray = values.clone().into_array(); let packed_with_patches = BitPackedData::encode(&parray, 9).unwrap(); - assert!(packed_with_patches.patches().is_some()); + assert!( + packed_with_patches + .patches(packed_with_patches.len()) + .is_some() + ); assert_arrays_eq!( packed_with_patches.as_array().to_primitive(), PrimitiveArray::new(values, vortex_array::validity::Validity::NonNullable) diff --git a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs index d14d218401c..80a73f82678 100644 --- a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs +++ b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs @@ -11,6 +11,8 @@ use lending_iterator::prelude::Item; use lending_iterator::prelude::LendingIterator; use vortex_array::dtype::PhysicalPType; use vortex_buffer::ByteBuffer; +use vortex_error::VortexResult; +use vortex_error::vortex_ensure; use crate::BitPackedData; @@ -55,10 +57,11 @@ impl> UnpackStrategy for BitPackingStr /// use vortex_array::IntoArray; /// use vortex_buffer::buffer; /// use vortex_fastlanes::BitPackedData; +/// use vortex_fastlanes::BitPackedArrayExt; /// use vortex_fastlanes::unpack_iter::BitUnpackedChunks; /// /// let array = BitPackedData::encode(&buffer![2, 3, 4, 5].into_array(), 2).unwrap(); -/// let mut unpacked_chunks: BitUnpackedChunks = array.unpacked_chunks(); +/// let mut unpacked_chunks: BitUnpackedChunks = array.unpacked_chunks().unwrap(); /// /// if let Some(header) = unpacked_chunks.initial() { /// // handle partial initial chunk @@ -89,13 +92,13 @@ pub struct UnpackedChunks> { pub type BitUnpackedChunks = UnpackedChunks; impl BitUnpackedChunks { - pub fn new(array: &BitPackedData) -> Self { - Self::new_with_strategy( + pub fn try_new(array: &BitPackedData, len: usize) -> VortexResult { + Self::try_new_with_strategy( BitPackingStrategy, array.packed().clone().unwrap_host(), array.bit_width() as usize, array.offset() as usize, - array.len(), + len, ) } @@ -115,26 +118,29 @@ impl BitUnpackedChunks { } impl> UnpackedChunks { - pub fn new_with_strategy( + pub fn try_new_with_strategy( strategy: S, packed: ByteBuffer, bit_width: usize, offset: usize, len: usize, - ) -> Self { + ) -> VortexResult { + vortex_ensure!( + offset < CHUNK_SIZE, + "Invalid bit-packed offset {offset}, expected < {CHUNK_SIZE}" + ); let elems_per_chunk = 128 * bit_width / size_of::(); let num_chunks = (offset + len).div_ceil(CHUNK_SIZE); - assert_eq!( - packed.len() / size_of::(), - num_chunks * elems_per_chunk, + vortex_ensure!( + packed.len() / size_of::() == num_chunks * elems_per_chunk, "Invalid packed length: got {}, expected {}", packed.len() / size_of::(), num_chunks * elems_per_chunk ); let last_chunk_length = (offset + len) % CHUNK_SIZE; - Self { + Ok(Self { strategy, bit_width, offset, @@ -143,7 +149,7 @@ impl> UnpackedChunks { buffer: [const { MaybeUninit::::uninit() }; CHUNK_SIZE], num_chunks, last_chunk_length, - } + }) } #[inline(always)] diff --git a/encodings/fastlanes/src/bitpacking/compute/cast.rs b/encodings/fastlanes/src/bitpacking/compute/cast.rs index 43cbc38d455..b7e6a7b63da 100644 --- a/encodings/fastlanes/src/bitpacking/compute/cast.rs +++ b/encodings/fastlanes/src/bitpacking/compute/cast.rs @@ -10,21 +10,20 @@ use vortex_array::patches::Patches; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; -use crate::BitPackedData; use crate::bitpacking::BitPacked; impl CastReduce for BitPacked { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { if array.dtype().eq_ignore_nullability(dtype) { let new_validity = array - .validity() + .validity(array.dtype().nullability()) .cast_nullability(dtype.nullability(), array.len())?; return Ok(Some( - BitPackedData::try_new( + BitPacked::try_new( array.packed().clone(), dtype.as_ptype(), new_validity, array - .patches() + .patches(array.len()) .map(|patches| { let new_values = patches.values().cast(dtype.clone())?; Patches::new( diff --git a/encodings/fastlanes/src/bitpacking/compute/filter.rs b/encodings/fastlanes/src/bitpacking/compute/filter.rs index a2887615698..5ebeaa4705c 100644 --- a/encodings/fastlanes/src/bitpacking/compute/filter.rs +++ b/encodings/fastlanes/src/bitpacking/compute/filter.rs @@ -25,6 +25,7 @@ use vortex_mask::MaskValues; use super::chunked_indices; use super::take::UNPACK_CHUNK_THRESHOLD; use crate::BitPacked; +use crate::BitPackedArrayExt; use crate::BitPackedData; /// The threshold over which it is faster to fully unpack the entire [`BitPackedArray`] and then @@ -58,16 +59,26 @@ impl FilterKernel for BitPacked { // If the density is high enough, then we would rather decompress the whole array and then apply // a filter over decompressing values one by one. - if values.density() > unpack_then_filter_threshold(array.ptype()) { + if values.density() > unpack_then_filter_threshold(array.dtype().as_ptype()) { return Ok(None); } // Filter and patch using the correct unsigned type for FastLanes, then cast to signed if needed. - let primitive = match_each_unsigned_integer_ptype!(array.ptype().to_unsigned(), |U| { - let (buffer, validity) = filter_primitive_without_patches::(&array, values)?; - // reinterpret_cast for signed types. - PrimitiveArray::new(buffer, validity).reinterpret_cast(array.ptype()) - }); + let primitive = + match_each_unsigned_integer_ptype!(array.dtype().as_ptype().to_unsigned(), |U| { + let (buffer, validity) = filter_primitive_without_patches::(array, values)?; + // reinterpret_cast for signed types. + let primitive = PrimitiveArray::new(buffer, validity); + if array.dtype().as_ptype().is_signed_int() { + PrimitiveArray::from_buffer_handle( + primitive.buffer_handle().clone(), + array.dtype().as_ptype(), + primitive.validity(), + ) + } else { + primitive + } + }); let patches = array .patches() @@ -76,12 +87,12 @@ impl FilterKernel for BitPacked { .flatten(); if let Some(patches) = patches { - let mut prim_array = PrimitiveArray::try_from_data(primitive)?; + let mut prim_array = primitive; prim_array = prim_array.patch(&patches, ctx)?; return Ok(Some(prim_array.into_array())); } - Ok(Some(PrimitiveArray::try_from_data(primitive)?.into_array())) + Ok(Some(primitive.into_array())) } } @@ -97,10 +108,10 @@ impl FilterKernel for BitPacked { /// /// Returns a tuple of (values buffer, validity mask). fn filter_primitive_without_patches( - array: &BitPackedData, + array: ArrayView<'_, BitPacked>, selection: &Arc, ) -> VortexResult<(Buffer, Validity)> { - let values = filter_with_indices(array, selection.indices()); + let values = filter_with_indices(array.data(), selection.indices()); let validity = array.validity().filter(&Mask::Values(selection.clone()))?; Ok((values.freeze(), validity)) @@ -264,7 +275,7 @@ mod test { let unpacked = PrimitiveArray::from_iter(values.clone()); let bitpacked = BitPackedData::encode(&unpacked.into_array(), 7).unwrap(); assert!( - bitpacked.patches().is_some(), + bitpacked.patches(bitpacked.len()).is_some(), "Expected patches for values exceeding bit width" ); @@ -296,7 +307,7 @@ mod test { let unpacked = PrimitiveArray::from_iter(values.clone()); let bitpacked = BitPackedData::encode(&unpacked.into_array(), 7).unwrap(); assert!( - bitpacked.patches().is_some(), + bitpacked.patches(bitpacked.len()).is_some(), "Expected patches for values exceeding bit width" ); diff --git a/encodings/fastlanes/src/bitpacking/compute/is_constant.rs b/encodings/fastlanes/src/bitpacking/compute/is_constant.rs index 314e4418ea1..900284e926d 100644 --- a/encodings/fastlanes/src/bitpacking/compute/is_constant.rs +++ b/encodings/fastlanes/src/bitpacking/compute/is_constant.rs @@ -22,6 +22,7 @@ use vortex_array::scalar::Scalar; use vortex_error::VortexResult; use crate::BitPacked; +use crate::BitPackedArrayExt; use crate::unpack_iter::BitPacked as BitPackedUnpack; /// BitPacked-specific is_constant kernel with SIMD support. @@ -43,7 +44,7 @@ impl DynAggregateKernel for BitPackedIsConstantKernel { return Ok(None); }; - let result = match_each_integer_ptype!(array.ptype(), |P| { + let result = match_each_integer_ptype!(array.dtype().as_ptype(), |P| { bitpacked_is_constant::() }>(array)? }); @@ -54,7 +55,7 @@ impl DynAggregateKernel for BitPackedIsConstantKernel { fn bitpacked_is_constant( array: ArrayView<'_, BitPacked>, ) -> VortexResult { - let mut bit_unpack_iterator = array.unpacked_chunks::(); + let mut bit_unpack_iterator = array.unpacked_chunks::()?; let patches = array.patches().map(|p| { let values = p.values().to_primitive(); let indices = p.indices().to_primitive(); diff --git a/encodings/fastlanes/src/bitpacking/compute/slice.rs b/encodings/fastlanes/src/bitpacking/compute/slice.rs index fe702e524b0..cc9394953f4 100644 --- a/encodings/fastlanes/src/bitpacking/compute/slice.rs +++ b/encodings/fastlanes/src/bitpacking/compute/slice.rs @@ -11,7 +11,6 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::BitPacked; -use crate::BitPackedData; impl SliceReduce for BitPacked { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { @@ -24,24 +23,24 @@ impl SliceReduce for BitPacked { let encoded_start = (block_start / 8) * array.bit_width() as usize; let encoded_stop = (block_stop / 8) * array.bit_width() as usize; - // slice the buffer using the encoded start/stop values - // SAFETY: slicing packed values without decoding preserves invariants - Ok(Some(unsafe { - BitPackedData::new_unchecked( + Ok(Some( + BitPacked::try_new( array.packed().slice(encoded_start..encoded_stop), - array.dtype().clone(), - array.validity().slice(range.clone())?, + array.dtype().as_ptype(), array - .patches() + .validity(array.dtype().nullability()) + .slice(range.clone())?, + array + .patches(array.len()) .map(|p| p.slice(range.clone())) .transpose()? .flatten(), array.bit_width(), range.len(), offset as u16, - ) - .into_array() - })) + )? + .into_array(), + )) } } diff --git a/encodings/fastlanes/src/bitpacking/compute/take.rs b/encodings/fastlanes/src/bitpacking/compute/take.rs index bf73dd1ba05..6d652c42b42 100644 --- a/encodings/fastlanes/src/bitpacking/compute/take.rs +++ b/encodings/fastlanes/src/bitpacking/compute/take.rs @@ -24,7 +24,7 @@ use vortex_error::VortexResult; use super::chunked_indices; use crate::BitPacked; -use crate::BitPackedData; +use crate::BitPackedArrayExt; use crate::bitpack_decompress; // TODO(connor): This is duplicated in `encodings/fastlanes/src/bitpacking/kernels/mod.rs`. @@ -54,15 +54,24 @@ impl TakeExecute for BitPacked { let indices = indices.clone().execute::(ctx)?; let taken = match_each_unsigned_integer_ptype!(ptype.to_unsigned(), |T| { match_each_integer_ptype!(indices.ptype(), |I| { - take_primitive::(&array, &indices, taken_validity, ctx)? + take_primitive::(array, &indices, taken_validity, ctx)? }) }); - Ok(Some(taken.reinterpret_cast(ptype).into_array())) + let taken = if ptype.is_signed_int() { + PrimitiveArray::from_buffer_handle( + taken.buffer_handle().clone(), + ptype, + taken.validity(), + ) + } else { + taken + }; + Ok(Some(taken.into_array())) } } fn take_primitive( - array: &BitPackedData, + array: ArrayView<'_, BitPacked>, indices: &PrimitiveArray, taken_validity: Validity, ctx: &mut ExecutionCtx, @@ -128,11 +137,13 @@ fn take_primitive( } }); - let unpatched_taken = if array.ptype().is_signed_int() { - // Flip back to signed type before patching. - PrimitiveArray::try_from_data( - PrimitiveArray::new(output, taken_validity).reinterpret_cast(array.ptype()), - )? + let unpatched_taken = if array.dtype().as_ptype().is_signed_int() { + let primitive = PrimitiveArray::new(output, taken_validity); + PrimitiveArray::from_buffer_handle( + primitive.buffer_handle().clone(), + array.dtype().as_ptype(), + primitive.validity(), + ) } else { PrimitiveArray::new(output, taken_validity) }; @@ -213,7 +224,7 @@ mod test { let values = (0..u16::MAX as u32 + num_patches as u32).collect::>(); let uncompressed = PrimitiveArray::new(values.clone(), Validity::NonNullable); let packed = BitPackedData::encode(&uncompressed.into_array(), 16).unwrap(); - assert!(packed.patches().is_some()); + assert!(packed.patches(packed.len()).is_some()); let rng = rng(); let range = Uniform::new(0, values.len()).unwrap(); @@ -245,7 +256,7 @@ mod test { BitPackedData::encode(&buffer![1i32, 2i32, 3i32, 4i32].into_array(), 1).unwrap(); let taken_primitive = take_primitive::( - &start, + start.as_view(), &PrimitiveArray::from_iter([0u64, 1, 2, 3]), Validity::NonNullable, &mut LEGACY_SESSION.create_execution_ctx(), diff --git a/encodings/fastlanes/src/bitpacking/mod.rs b/encodings/fastlanes/src/bitpacking/mod.rs index 5a3a6ed0200..95d8ef8c608 100644 --- a/encodings/fastlanes/src/bitpacking/mod.rs +++ b/encodings/fastlanes/src/bitpacking/mod.rs @@ -2,8 +2,9 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::BitPackedArrayParts; +pub use array::BitPackedArrayExt; pub use array::BitPackedData; +pub use array::BitPackedDataParts; pub use array::bitpack_compress; pub use array::bitpack_decompress; pub use array::unpack_iter; diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs index 1e7ccd08665..317788ce9c5 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs @@ -3,20 +3,19 @@ use std::hash::Hash; +use prost::Message; use vortex_array::AnyCanonical; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; -use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; -use vortex_array::SerializeMetadata; use vortex_array::buffer::BufferHandle; use vortex_array::builders::ArrayBuilder; use vortex_array::dtype::DType; @@ -27,7 +26,6 @@ use vortex_array::patches::PatchesMetadata; use vortex_array::require_patches; use vortex_array::require_validity; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; @@ -39,7 +37,9 @@ use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use crate::BitPackedArrayExt; use crate::BitPackedData; +use crate::BitPackedDataParts; use crate::bitpack_decompress::unpack_array; use crate::bitpack_decompress::unpack_into_primitive_builder; use crate::bitpacking::array::NUM_SLOTS; @@ -70,29 +70,15 @@ pub struct BitPackedMetadata { impl VTable for BitPacked { type ArrayData = BitPackedData; - type Metadata = ProstMetadata; - type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &BitPackedData) -> &Self { - &BitPacked - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &BitPackedData) -> usize { - array.len - } - - fn dtype(array: &BitPackedData) -> &DType { - &array.dtype - } - - fn stats(array: &BitPackedData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate_against_outer(dtype, len) } fn array_hash( @@ -103,16 +89,64 @@ impl VTable for BitPacked { array.offset.hash(state); array.bit_width.hash(state); array.packed.array_hash(state, precision); - array.patches().array_hash(state, precision); - array.validity().array_hash(state, precision); + match array.patch_indices() { + Some(indices) => { + true.hash(state); + indices.array_hash(state, precision); + } + None => false.hash(state), + } + match array.patch_values() { + Some(values) => { + true.hash(state); + values.array_hash(state, precision); + } + None => false.hash(state), + } + match array.patch_chunk_offsets() { + Some(offsets) => { + true.hash(state); + offsets.array_hash(state, precision); + } + None => false.hash(state), + } + match array.validity_child() { + Some(validity) => { + true.hash(state); + validity.array_hash(state, precision); + } + None => false.hash(state), + } + array.patch_offset.hash(state); + array.patch_offset_within_chunk.hash(state); } fn array_eq(array: &BitPackedData, other: &BitPackedData, precision: Precision) -> bool { array.offset == other.offset && array.bit_width == other.bit_width && array.packed.array_eq(&other.packed, precision) - && array.patches().array_eq(&other.patches(), precision) - && array.validity().array_eq(&other.validity(), precision) + && match (array.patch_indices(), other.patch_indices()) { + (Some(lhs), Some(rhs)) => lhs.array_eq(rhs, precision), + (None, None) => true, + _ => false, + } + && match (array.patch_values(), other.patch_values()) { + (Some(lhs), Some(rhs)) => lhs.array_eq(rhs, precision), + (None, None) => true, + _ => false, + } + && match (array.patch_chunk_offsets(), other.patch_chunk_offsets()) { + (Some(lhs), Some(rhs)) => lhs.array_eq(rhs, precision), + (None, None) => true, + _ => false, + } + && match (array.validity_child(), other.validity_child()) { + (Some(lhs), Some(rhs)) => lhs.array_eq(rhs, precision), + (None, None) => true, + _ => false, + } + && array.patch_offset == other.patch_offset + && array.patch_offset_within_chunk == other.patch_offset_within_chunk } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -167,44 +201,30 @@ impl VTable for BitPacked { Ok(()) } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(BitPackedMetadata { - bit_width: array.bit_width() as u32, - offset: array.offset() as u32, - patches: array - .patches() - .map(|p| p.to_metadata(array.len(), array.dtype())) - .transpose()?, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some( + BitPackedMetadata { + bit_width: array.bit_width() as u32, + offset: array.offset() as u32, + patches: array + .patches() + .map(|p| p.to_metadata(array.len(), array.dtype())) + .transpose()?, + } + .encode_to_vec(), + )) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - let inner = as DeserializeMetadata>::deserialize(bytes)?; - Ok(ProstMetadata(inner)) - } - - /// Deserialize a BitPackedArray from its components. - /// - /// Note that the layout depends on whether patches and chunk_offsets are present: - /// - No patches: `[validity?]` - /// - With patches: `[patch_indices, patch_values, chunk_offsets?, validity?]` - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = BitPackedMetadata::decode(metadata)?; if buffers.len() != 1 { vortex_bail!("Expected 1 buffer, got {}", buffers.len()); } @@ -274,9 +294,9 @@ impl VTable for BitPacked { builder: &mut dyn ArrayBuilder, ctx: &mut ExecutionCtx, ) -> VortexResult<()> { - match_each_integer_ptype!(array.ptype(), |T| { + match_each_integer_ptype!(array.dtype().as_ptype(), |T| { unpack_into_primitive_builder::( - &array, + array, builder .as_any_mut() .downcast_mut() @@ -294,10 +314,14 @@ impl VTable for BitPacked { PATCH_VALUES_SLOT, PATCH_CHUNK_OFFSETS_SLOT ); - require_validity!(array, &array.validity(), VALIDITY_SLOT => AnyCanonical); + require_validity!( + array, + &array.validity(), + VALIDITY_SLOT => AnyCanonical + ); Ok(ExecutionResult::done( - unpack_array(&array, ctx)?.into_array(), + unpack_array(array.as_view(), ctx)?.into_array(), )) } @@ -317,6 +341,27 @@ pub struct BitPacked; impl BitPacked { pub const ID: ArrayId = ArrayId::new_ref("fastlanes.bitpacked"); + pub fn try_new( + packed: BufferHandle, + ptype: PType, + validity: Validity, + patches: Option, + bit_width: u8, + len: usize, + offset: u16, + ) -> VortexResult { + let dtype = DType::Primitive(ptype, validity.nullability()); + let data = + BitPackedData::try_new(packed, ptype, validity, patches, bit_width, len, offset)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(BitPacked, dtype, len, data)) }) + } + + pub fn into_parts(array: BitPackedArray) -> BitPackedDataParts { + let len = array.len(); + let nullability = array.dtype().nullability(); + array.into_data().into_parts(len, nullability) + } + /// Encode an array into a bitpacked representation with the given bit width. pub fn encode(array: &ArrayRef, bit_width: u8) -> VortexResult { BitPackedData::encode(array, bit_width) diff --git a/encodings/fastlanes/src/bitpacking/vtable/operations.rs b/encodings/fastlanes/src/bitpacking/vtable/operations.rs index 8dcfa18d08d..49984c80ade 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/operations.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/operations.rs @@ -16,12 +16,12 @@ impl OperationsVTable for BitPacked { _ctx: &mut ExecutionCtx, ) -> VortexResult { Ok( - if let Some(patches) = array.patches() + if let Some(patches) = array.patches(array.len()) && let Some(patch) = patches.get_patched(index)? { patch } else { - bitpack_decompress::unpack_single(&array, index) + bitpack_decompress::unpack_single(array, index) }, ) } @@ -141,14 +141,14 @@ mod test { // We create an array that has 1 element that does not fit in the 6-bit range. let array = BitPackedData::encode(&buffer![0u32..=64].into_array(), 6).unwrap(); - assert!(array.patches().is_some()); + assert!(array.patches(array.len()).is_some()); - let patch_indices = array.patches().unwrap().indices().clone(); + let patch_indices = array.patches(array.len()).unwrap().indices().clone(); assert_eq!(patch_indices.len(), 1); // Slicing drops the empty patches array. let sliced_bp = slice_via_reduce(&array, 0..64); - assert!(sliced_bp.patches().is_none()); + assert!(sliced_bp.patches(sliced_bp.len()).is_none()); } #[test] @@ -177,30 +177,29 @@ mod test { #[test] fn scalar_at_invalid_patches() { - let packed_array = unsafe { - BitPackedData::new_unchecked( - BufferHandle::new_host(ByteBuffer::copy_from_aligned( - [0u8; 128], - Alignment::of::(), - )), - DType::Primitive(PType::U32, true.into()), - Validity::AllInvalid, - Some( - Patches::new( - 8, - 0, - buffer![1u32].into_array(), - PrimitiveArray::new(buffer![999u32], Validity::AllValid).into_array(), - None, - ) - .unwrap(), - ), - 1, - 8, - 0, - ) - .into_array() - }; + let packed_array = BitPacked::try_new( + BufferHandle::new_host(ByteBuffer::copy_from_aligned( + [0u8; 128], + Alignment::of::(), + )), + PType::U32, + Validity::AllInvalid, + Some( + Patches::new( + 8, + 0, + buffer![1u32].into_array(), + PrimitiveArray::new(buffer![999u32], Validity::AllValid).into_array(), + None, + ) + .unwrap(), + ), + 1, + 8, + 0, + ) + .unwrap() + .into_array(); assert_eq!( packed_array.scalar_at(1).unwrap(), Scalar::null(DType::Primitive(PType::U32, Nullability::Nullable)) @@ -212,9 +211,9 @@ mod test { let values = (0u32..257).collect::>(); let uncompressed = values.clone().into_array(); let packed = BitPackedData::encode(&uncompressed, 8).unwrap(); - assert!(packed.patches().is_some()); + assert!(packed.patches(packed.len()).is_some()); - let patches = packed.patches().unwrap().indices().clone(); + let patches = packed.patches(packed.len()).unwrap().indices().clone(); assert_eq!( usize::try_from(&patches.scalar_at(0).unwrap()).unwrap(), 256 diff --git a/encodings/fastlanes/src/bitpacking/vtable/validity.rs b/encodings/fastlanes/src/bitpacking/vtable/validity.rs index c9b555eb92c..2313e0112eb 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/validity.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/validity.rs @@ -10,6 +10,6 @@ use crate::BitPacked; impl ValidityVTable for BitPacked { fn validity(array: ArrayView<'_, BitPacked>) -> VortexResult { - Ok(array.data().validity()) + Ok(array.data().validity(array.dtype().nullability())) } } diff --git a/encodings/fastlanes/src/delta/array/delta_compress.rs b/encodings/fastlanes/src/delta/array/delta_compress.rs index 3f7a75d3a82..0b79489246d 100644 --- a/encodings/fastlanes/src/delta/array/delta_compress.rs +++ b/encodings/fastlanes/src/delta/array/delta_compress.rs @@ -104,8 +104,7 @@ mod tests { use vortex_error::VortexResult; use vortex_session::VortexSession; - use crate::DeltaArray; - use crate::DeltaData; + use crate::Delta; use crate::bitpack_compress::bitpack_encode; use crate::delta::array::delta_decompress::delta_decompress; use crate::delta_compress; @@ -120,10 +119,7 @@ mod tests { (0u32..10_000).map(|i| (i % 2 == 0).then_some(i)), ))] fn test_compress(#[case] array: PrimitiveArray) -> VortexResult<()> { - let delta = DeltaArray::try_from_data(DeltaData::try_from_primitive_array( - &array, - &mut SESSION.create_execution_ctx(), - )?)?; + let delta = Delta::try_from_primitive_array(&array, &mut SESSION.create_execution_ctx())?; assert_eq!(delta.len(), array.len()); let decompressed = delta_decompress(&delta, &mut SESSION.create_execution_ctx())?; assert_arrays_eq!(decompressed, array); @@ -140,16 +136,13 @@ mod tests { ); let (bases, deltas) = delta_compress(&array, &mut SESSION.create_execution_ctx()).unwrap(); let bitpacked_deltas = bitpack_encode(&deltas, 1, None).unwrap(); - let packed_delta = DeltaArray::try_from_data( - DeltaData::try_new( - bases.into_array(), - bitpacked_deltas.into_array(), - 0, - array.len(), - ) - .unwrap(), + let packed_delta = Delta::try_new( + bases.into_array(), + bitpacked_deltas.into_array(), + 0, + array.len(), ) - .vortex_expect("DeltaData is always valid"); + .vortex_expect("Delta array construction should succeed"); assert_arrays_eq!(packed_delta.as_array().to_primitive(), array); } } diff --git a/encodings/fastlanes/src/delta/array/mod.rs b/encodings/fastlanes/src/delta/array/mod.rs index 048b860e000..05d9dfbbadb 100644 --- a/encodings/fastlanes/src/delta/array/mod.rs +++ b/encodings/fastlanes/src/delta/array/mod.rs @@ -3,13 +3,9 @@ use fastlanes::FastLanes; use vortex_array::ArrayRef; -use vortex_array::ExecutionCtx; -use vortex_array::IntoArray; -use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_array::match_each_unsigned_integer_ptype; -use vortex_array::stats::ArrayStats; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; @@ -37,11 +33,11 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["bases", "deltas"]; /// use vortex_array::VortexSessionExecute; /// use vortex_array::session::ArraySession; /// use vortex_session::VortexSession; -/// use vortex_fastlanes::DeltaData; +/// use vortex_fastlanes::Delta; /// /// let session = VortexSession::empty().with::(); /// let primitive = PrimitiveArray::from_iter([1_u32, 2, 3, 5, 10, 11]); -/// let array = DeltaData::try_from_primitive_array(&primitive, &mut session.create_execution_ctx()).unwrap(); +/// let array = Delta::try_from_primitive_array(&primitive, &mut session.create_execution_ctx()).unwrap(); /// ``` /// /// # Details @@ -66,23 +62,10 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["bases", "deltas"]; #[derive(Clone, Debug)] pub struct DeltaData { pub(super) offset: usize, - pub(super) len: usize, - pub(super) dtype: DType, pub(super) slots: Vec>, - pub(super) stats_set: ArrayStats, } impl DeltaData { - pub fn try_from_primitive_array( - array: &PrimitiveArray, - ctx: &mut ExecutionCtx, - ) -> VortexResult { - let logical_len = array.len(); - let (bases, deltas) = delta_compress::delta_compress(array, ctx)?; - - Self::try_new(bases.into_array(), deltas.into_array(), 0, logical_len) - } - /// Create a DeltaArray from the given `bases` and `deltas` arrays /// with given `offset` into first chunk and `logical_len` length. pub fn try_new( @@ -91,6 +74,31 @@ impl DeltaData { offset: usize, len: usize, ) -> VortexResult { + Self::validate_parts(&bases, &deltas, offset, len)?; + + // SAFETY: validation done above + Ok(unsafe { Self::new_unchecked(bases, deltas, offset) }) + } + + pub(crate) fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { + Self::validate_parts(self.bases(), self.deltas(), self.offset, len)?; + let expected_dtype = self + .bases() + .dtype() + .with_nullability(self.deltas().dtype().nullability()); + vortex_ensure!( + dtype == &expected_dtype, + "DeltaArray dtype mismatch: expected {expected_dtype}, got {dtype}" + ); + Ok(()) + } + + fn validate_parts( + bases: &ArrayRef, + deltas: &ArrayRef, + offset: usize, + len: usize, + ) -> VortexResult<()> { vortex_ensure!(offset < 1024, "offset must be less than 1024: {offset}"); vortex_ensure!( offset + len <= deltas.len(), @@ -105,8 +113,8 @@ impl DeltaData { ); vortex_ensure!( - bases.dtype().is_int(), - "DeltaArray: dtype must be an integer, got {}", + bases.dtype().is_unsigned_int(), + "DeltaArray: dtype must be an unsigned integer, got {}", bases.dtype() ); @@ -122,24 +130,13 @@ impl DeltaData { "bases length ({}) must be a multiple of LANES ({lanes})", bases.len(), ); - - // SAFETY: validation done above - Ok(unsafe { Self::new_unchecked(bases, deltas, offset, len) }) + Ok(()) } - pub(crate) unsafe fn new_unchecked( - bases: ArrayRef, - deltas: ArrayRef, - offset: usize, - logical_len: usize, - ) -> Self { - let dtype = bases.dtype().with_nullability(deltas.dtype().nullability()); + pub(crate) unsafe fn new_unchecked(bases: ArrayRef, deltas: ArrayRef, offset: usize) -> Self { Self { offset, - len: logical_len, - dtype, slots: vec![Some(bases), Some(deltas)], - stats_set: Default::default(), } } @@ -157,25 +154,6 @@ impl DeltaData { .vortex_expect("DeltaArray deltas slot") } - pub(crate) fn lanes(&self) -> usize { - lane_count(self.dtype().as_ptype()) - } - - #[inline] - pub fn len(&self) -> usize { - self.len - } - - #[inline] - pub fn is_empty(&self) -> bool { - self.len == 0 - } - - #[inline] - pub fn dtype(&self) -> &DType { - &self.dtype - } - #[inline] /// The logical offset into the first chunk of [`Self::deltas`]. pub fn offset(&self) -> usize { @@ -189,10 +167,6 @@ impl DeltaData { pub(crate) fn deltas_len(&self) -> usize { self.deltas().len() } - - pub(crate) fn stats_set(&self) -> &ArrayStats { - &self.stats_set - } } pub(crate) fn lane_count(ptype: PType) -> usize { diff --git a/encodings/fastlanes/src/delta/compute/cast.rs b/encodings/fastlanes/src/delta/compute/cast.rs index 46575c5dc6b..a9af6d09d60 100644 --- a/encodings/fastlanes/src/delta/compute/cast.rs +++ b/encodings/fastlanes/src/delta/compute/cast.rs @@ -11,7 +11,6 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use vortex_error::vortex_panic; -use crate::DeltaData; use crate::delta::Delta; impl CastReduce for Delta { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { @@ -38,8 +37,7 @@ impl CastReduce for Delta { // Create a new DeltaArray with the casted components, preserving offset and logical length Ok(Some( - DeltaData::try_new(casted_bases, casted_deltas, array.offset(), array.len())? - .into_array(), + Delta::try_new(casted_bases, casted_deltas, array.offset(), array.len())?.into_array(), )) } } @@ -62,7 +60,7 @@ mod tests { use vortex_buffer::buffer; use vortex_session::VortexSession; - use crate::DeltaData; + use crate::Delta; static SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); @@ -70,7 +68,7 @@ mod tests { fn test_cast_delta_u8_to_u32() { let primitive = PrimitiveArray::from_iter([10u8, 20, 30, 40, 50]); let array = - DeltaData::try_from_primitive_array(&primitive, &mut SESSION.create_execution_ctx()) + Delta::try_from_primitive_array(&primitive, &mut SESSION.create_execution_ctx()) .unwrap(); let casted = array @@ -95,8 +93,7 @@ mod tests { vortex_array::validity::Validity::NonNullable, ); let array = - DeltaData::try_from_primitive_array(&values, &mut SESSION.create_execution_ctx()) - .unwrap(); + Delta::try_from_primitive_array(&values, &mut SESSION.create_execution_ctx()).unwrap(); let casted = array .into_array() @@ -135,7 +132,7 @@ mod tests { )] fn test_cast_delta_conformance(#[case] primitive: PrimitiveArray) { let delta_array = - DeltaData::try_from_primitive_array(&primitive, &mut SESSION.create_execution_ctx()) + Delta::try_from_primitive_array(&primitive, &mut SESSION.create_execution_ctx()) .unwrap(); test_cast_conformance(&delta_array.into_array()); } diff --git a/encodings/fastlanes/src/delta/vtable/mod.rs b/encodings/fastlanes/src/delta/vtable/mod.rs index 887de327a82..746221a5b1e 100644 --- a/encodings/fastlanes/src/delta/vtable/mod.rs +++ b/encodings/fastlanes/src/delta/vtable/mod.rs @@ -9,20 +9,19 @@ use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_array::match_each_unsigned_integer_ptype; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::VTable; use vortex_error::VortexResult; @@ -55,29 +54,15 @@ pub struct DeltaMetadata { impl VTable for Delta { type ArrayData = DeltaData; - type Metadata = ProstMetadata; - type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &DeltaData) -> &Self { - &Delta - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &DeltaData) -> usize { - array.len() - } - - fn dtype(array: &DeltaData) -> &DType { - array.dtype() - } - - fn stats(array: &DeltaData) -> &ArrayStats { - array.stats_set() + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate(dtype, len) } fn array_hash(array: &DeltaData, state: &mut H, precision: Precision) { @@ -131,41 +116,42 @@ impl VTable for Delta { Ok(()) } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(DeltaMetadata { - deltas_len: array.deltas().len() as u64, - offset: array.offset() as u32, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.0.encode_to_vec())) + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some( + DeltaMetadata { + deltas_len: array.deltas().len() as u64, + offset: array.offset() as u32, + } + .encode_to_vec(), + )) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata(DeltaMetadata::decode(bytes)?)) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, - _buffers: &[BufferHandle], + metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { - assert_eq!(children.len(), 2); + vortex_ensure!( + buffers.is_empty(), + "DeltaArray expects 0 buffers, got {}", + buffers.len() + ); + vortex_ensure!( + children.len() == 2, + "DeltaArray expects 2 children, got {}", + children.len() + ); + let metadata = DeltaMetadata::decode(metadata)?; let ptype = PType::try_from(dtype)?; let lanes = match_each_unsigned_integer_ptype!(ptype, |T| { ::LANES }); // Compute the length of the bases array - let deltas_len = usize::try_from(metadata.0.deltas_len) - .map_err(|_| vortex_err!("deltas_len {} overflowed usize", metadata.0.deltas_len))?; + let deltas_len = usize::try_from(metadata.deltas_len) + .map_err(|_| vortex_err!("deltas_len {} overflowed usize", metadata.deltas_len))?; let num_chunks = deltas_len / 1024; let remainder_base_size = if deltas_len % 1024 > 0 { 1 } else { 0 }; let bases_len = num_chunks * lanes + remainder_base_size; @@ -173,7 +159,7 @@ impl VTable for Delta { let bases = children.get(0, dtype, bases_len)?; let deltas = children.get(1, dtype, deltas_len)?; - DeltaData::try_new(bases, deltas, metadata.0.offset as usize, len) + DeltaData::try_new(bases, deltas, metadata.offset as usize, len) } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { @@ -189,31 +175,45 @@ pub struct Delta; impl Delta { pub const ID: ArrayId = ArrayId::new_ref("fastlanes.delta"); + pub fn try_new( + bases: ArrayRef, + deltas: ArrayRef, + offset: usize, + len: usize, + ) -> VortexResult { + let dtype = bases.dtype().with_nullability(deltas.dtype().nullability()); + let data = DeltaData::try_new(bases, deltas, offset, len)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Delta, dtype, len, data)) }) + } + /// Compress a primitive array using Delta encoding. pub fn try_from_primitive_array( array: &PrimitiveArray, ctx: &mut ExecutionCtx, ) -> VortexResult { - Array::try_from_data(DeltaData::try_from_primitive_array(array, ctx)?) + let logical_len = array.len(); + let (bases, deltas) = crate::delta::array::delta_compress::delta_compress(array, ctx)?; + Self::try_new(bases.into_array(), deltas.into_array(), 0, logical_len) } } #[cfg(test)] mod tests { + use prost::Message; use vortex_array::test_harness::check_metadata; use super::DeltaMetadata; - use super::ProstMetadata; #[cfg_attr(miri, ignore)] #[test] fn test_delta_metadata() { check_metadata( "delta.metadata", - ProstMetadata(DeltaMetadata { + &DeltaMetadata { offset: u32::MAX, deltas_len: u64::MAX, - }), + } + .encode_to_vec(), ); } } diff --git a/encodings/fastlanes/src/delta/vtable/operations.rs b/encodings/fastlanes/src/delta/vtable/operations.rs index cae9b701ee9..731588a67bb 100644 --- a/encodings/fastlanes/src/delta/vtable/operations.rs +++ b/encodings/fastlanes/src/delta/vtable/operations.rs @@ -38,18 +38,15 @@ mod tests { use vortex_error::VortexExpect; use vortex_session::VortexSession; + use crate::Delta; use crate::DeltaArray; - use crate::DeltaData; static SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); fn da(array: &PrimitiveArray) -> DeltaArray { - DeltaArray::try_from_data( - DeltaData::try_from_primitive_array(array, &mut SESSION.create_execution_ctx()) - .unwrap(), - ) - .vortex_expect("DeltaData is always valid") + Delta::try_from_primitive_array(array, &mut SESSION.create_execution_ctx()) + .vortex_expect("Delta array construction should succeed") } #[test] diff --git a/encodings/fastlanes/src/delta/vtable/slice.rs b/encodings/fastlanes/src/delta/vtable/slice.rs index 3068569d3c0..c836bed33fd 100644 --- a/encodings/fastlanes/src/delta/vtable/slice.rs +++ b/encodings/fastlanes/src/delta/vtable/slice.rs @@ -10,7 +10,7 @@ use vortex_array::IntoArray; use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; -use crate::DeltaData; +use crate::delta::array::lane_count; use crate::delta::vtable::Delta; impl SliceReduce for Delta { @@ -23,7 +23,7 @@ impl SliceReduce for Delta { let bases = array.bases(); let deltas = array.deltas(); - let lanes = array.lanes(); + let lanes = lane_count(array.dtype().as_ptype()); let new_bases = bases.slice( min(start_chunk * lanes, array.bases_len())..min(stop_chunk * lanes, array.bases_len()), @@ -33,10 +33,8 @@ impl SliceReduce for Delta { min(start_chunk * 1024, array.deltas_len())..min(stop_chunk * 1024, array.deltas_len()), )?; - // SAFETY: slicing valid bases/deltas preserves correctness - Ok(Some(unsafe { - DeltaData::new_unchecked(new_bases, new_deltas, physical_start % 1024, range.len()) - .into_array() - })) + Ok(Some( + Delta::try_new(new_bases, new_deltas, physical_start % 1024, range.len())?.into_array(), + )) } } diff --git a/encodings/fastlanes/src/for/array/for_compress.rs b/encodings/fastlanes/src/for/array/for_compress.rs index b911e20fa17..86bb6de2947 100644 --- a/encodings/fastlanes/src/for/array/for_compress.rs +++ b/encodings/fastlanes/src/for/array/for_compress.rs @@ -5,20 +5,18 @@ use num_traits::PrimInt; use num_traits::WrappingSub; use vortex_array::IntoArray; use vortex_array::arrays::PrimitiveArray; -use vortex_array::arrays::primitive::PrimitiveData; use vortex_array::dtype::NativePType; use vortex_array::expr::stats::Stat; use vortex_array::match_each_integer_ptype; -use vortex_array::stats::ArrayStats; use vortex_error::VortexResult; use vortex_error::vortex_err; +use crate::FoR; use crate::FoRArray; use crate::FoRData; impl FoRData { pub fn encode(array: PrimitiveArray) -> VortexResult { let array_ref = array.clone().into_array(); - let stats = ArrayStats::from(array_ref.statistics().to_owned()); let min = array_ref .statistics() .compute_stat(Stat::Min)? @@ -27,24 +25,17 @@ impl FoRData { let encoded = match_each_integer_ptype!(array.ptype(), |T| { compress_primitive::(array, T::try_from(&min)?)?.into_array() }); - let for_data = FoRData::try_new(encoded, min)?; - let for_array = FoRArray::try_from_data(for_data)?; - let for_ref = for_array.clone().into_array(); - for_array - .stats_set() - .to_ref(&for_ref) - .inherit_from(stats.to_ref(&for_ref)); - Ok(for_array) + FoR::try_new(encoded, min) } } fn compress_primitive( parray: PrimitiveArray, min: T, -) -> VortexResult { +) -> VortexResult { // Set null values to the min value, ensuring that decompress into a value in the primitive // range (and stop them wrapping around). - parray + let encoded = parray .into_data() .map_each_with_validity::(|(v, bool)| { if bool { @@ -52,7 +43,12 @@ fn compress_primitive( } else { T::zero() } - }) + })?; + Ok(PrimitiveArray::from_buffer_handle( + encoded.buffer_handle().clone(), + encoded.ptype(), + encoded.validity(), + )) } #[cfg(test)] @@ -73,7 +69,6 @@ mod test { use super::*; use crate::BitPackedData; - use crate::FoRArray; use crate::r#for::array::for_decompress::decompress; use crate::r#for::array::for_decompress::fused_decompress; @@ -137,7 +132,7 @@ mod test { let expect = PrimitiveArray::from_iter((0u32..1024).map(|x| x % 7 + 10)); let array = PrimitiveArray::from_iter((0u32..1024).map(|x| x % 7)); let bp = BitPackedData::encode(&array.into_array(), 3).unwrap(); - let compressed = FoRData::try_new(bp.into_array(), 10u32.into()).unwrap(); + let compressed = FoR::try_new(bp.into_array(), 10u32.into()).unwrap(); assert_arrays_eq!(compressed, expect); } @@ -147,9 +142,7 @@ mod test { let expect = PrimitiveArray::from_iter((0u32..1024).map(|x| x % 7 + 10)); let array = PrimitiveArray::from_iter((0u32..1024).map(|x| x % 7)); let bp = BitPackedData::encode(&array.into_array(), 2).unwrap(); - let compressed = FoRArray::try_from_data( - FoRData::try_new(bp.clone().into_array(), 10u32.into()).unwrap(), - )?; + let compressed = FoR::try_new(bp.clone().into_array(), 10u32.into())?; let decompressed = fused_decompress::( &compressed, bp.as_view(), @@ -178,7 +171,7 @@ mod test { .reinterpret_cast(PType::U8); let unsigned: Vec = (0..=u8::MAX).collect_vec(); let expected_unsigned = PrimitiveArray::from_iter(unsigned); - assert_arrays_eq!(encoded, expected_unsigned); + assert_eq!(encoded.as_slice::(), expected_unsigned.as_slice::()); let decompressed = decompress(&compressed, &mut SESSION.create_execution_ctx())?; array diff --git a/encodings/fastlanes/src/for/array/for_decompress.rs b/encodings/fastlanes/src/for/array/for_decompress.rs index 13de40e6365..82a84b73fa4 100644 --- a/encodings/fastlanes/src/for/array/for_decompress.rs +++ b/encodings/fastlanes/src/for/array/for_decompress.rs @@ -18,6 +18,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::BitPacked; +use crate::BitPackedArrayExt; use crate::FoRArray; use crate::bitpack_decompress; use crate::unpack_iter::UnpackStrategy; @@ -92,13 +93,13 @@ pub(crate) fn fused_decompress< let strategy = FoRStrategy { reference: ref_ }; // Create [`UnpackedChunks`] with FoR strategy. - let mut unpacked = UnpackedChunks::new_with_strategy( + let mut unpacked = UnpackedChunks::try_new_with_strategy( strategy, bp.packed().as_host().clone(), bp.bit_width() as usize, bp.offset() as usize, bp.len(), - ); + )?; let mut builder = PrimitiveBuilder::::with_capacity( for_.reference_scalar().dtype().nullability(), diff --git a/encodings/fastlanes/src/for/array/mod.rs b/encodings/fastlanes/src/for/array/mod.rs index 74d88580bd3..f2cd3108250 100644 --- a/encodings/fastlanes/src/for/array/mod.rs +++ b/encodings/fastlanes/src/for/array/mod.rs @@ -5,10 +5,9 @@ use vortex_array::ArrayRef; use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; -use vortex_array::stats::ArrayStats; use vortex_error::VortexExpect as _; use vortex_error::VortexResult; -use vortex_error::vortex_bail; +use vortex_error::vortex_ensure; pub mod for_compress; pub mod for_decompress; @@ -26,33 +25,46 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["encoded"]; pub struct FoRData { pub(super) slots: Vec>, pub(super) reference: Scalar, - pub(super) stats_set: ArrayStats, } impl FoRData { - pub fn try_new(encoded: ArrayRef, reference: Scalar) -> VortexResult { - if reference.is_null() { - vortex_bail!("Reference value cannot be null"); - } - let reference = reference.cast( - &reference - .dtype() - .with_nullability(encoded.dtype().nullability()), - )?; + pub(crate) fn try_new(encoded: ArrayRef, reference: Scalar) -> VortexResult { + Self::validate_parts(&encoded, &reference, reference.dtype(), encoded.len())?; Ok(Self { slots: vec![Some(encoded)], reference, - stats_set: Default::default(), }) } - pub(crate) unsafe fn new_unchecked(encoded: ArrayRef, reference: Scalar) -> Self { - Self { - slots: vec![Some(encoded)], - reference, - stats_set: Default::default(), - } + pub(crate) fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { + Self::validate_parts(self.encoded(), &self.reference, dtype, len) + } + + fn validate_parts( + encoded: &ArrayRef, + reference: &Scalar, + dtype: &DType, + len: usize, + ) -> VortexResult<()> { + vortex_ensure!(!reference.is_null(), "Reference value cannot be null"); + vortex_ensure!(dtype.is_int(), "FoR requires an integer dtype, got {dtype}"); + vortex_ensure!( + reference.dtype() == dtype, + "FoR reference dtype mismatch: expected {dtype}, got {}", + reference.dtype() + ); + vortex_ensure!( + encoded.dtype() == dtype, + "FoR encoded dtype mismatch: expected {dtype}, got {}", + encoded.dtype() + ); + vortex_ensure!( + encoded.len() == len, + "FoR encoded length mismatch: expected {len}, got {}", + encoded.len() + ); + Ok(()) } /// Returns the length of the array. @@ -89,8 +101,4 @@ impl FoRData { pub fn reference_scalar(&self) -> &Scalar { &self.reference } - - pub(crate) fn stats_set(&self) -> &ArrayStats { - &self.stats_set - } } diff --git a/encodings/fastlanes/src/for/compute/cast.rs b/encodings/fastlanes/src/for/compute/cast.rs index 884a0a8b94f..e37b25b64a1 100644 --- a/encodings/fastlanes/src/for/compute/cast.rs +++ b/encodings/fastlanes/src/for/compute/cast.rs @@ -9,7 +9,6 @@ use vortex_array::dtype::DType; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; -use crate::FoRData; use crate::r#for::FoR; impl CastReduce for FoR { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { @@ -23,7 +22,7 @@ impl CastReduce for FoR { let casted_reference = array.reference_scalar().cast(dtype)?; Ok(Some( - FoRData::try_new(casted_child, casted_reference)?.into_array(), + FoR::try_new(casted_child, casted_reference)?.into_array(), )) } } @@ -44,12 +43,11 @@ mod tests { use vortex_buffer::buffer; use vortex_error::VortexExpect; + use crate::FoR; use crate::FoRArray; - use crate::FoRData; fn for_arr(encoded: ArrayRef, reference: Scalar) -> FoRArray { - FoRArray::try_from_data(FoRData::try_new(encoded, reference).unwrap()) - .vortex_expect("FoRData is always valid") + FoR::try_new(encoded, reference).vortex_expect("FoR array construction should succeed") } #[test] diff --git a/encodings/fastlanes/src/for/compute/compare.rs b/encodings/fastlanes/src/for/compute/compare.rs index f23ec8c0256..853bf4c2955 100644 --- a/encodings/fastlanes/src/for/compute/compare.rs +++ b/encodings/fastlanes/src/for/compute/compare.rs @@ -23,7 +23,6 @@ use vortex_error::VortexExpect as _; use vortex_error::VortexResult; use crate::FoR; -use crate::FoRData; impl CompareKernel for FoR { fn compare( @@ -37,7 +36,7 @@ impl CompareKernel for FoR { { match_each_integer_ptype!(constant.ptype(), |T| { return compare_constant( - &lhs, + lhs, constant .typed_value::() .vortex_expect("null scalar handled in adaptor"), @@ -52,7 +51,7 @@ impl CompareKernel for FoR { } fn compare_constant( - lhs: &FoRData, + lhs: ArrayView<'_, FoR>, mut rhs: T, nullability: Nullability, operator: CompareOperator, @@ -99,12 +98,11 @@ mod tests { use vortex_buffer::buffer; use super::*; + use crate::FoR; use crate::FoRArray; - use crate::FoRData; fn for_arr(encoded: ArrayRef, reference: Scalar) -> FoRArray { - FoRArray::try_from_data(FoRData::try_new(encoded, reference).unwrap()) - .vortex_expect("FoRData is always valid") + FoR::try_new(encoded, reference).vortex_expect("FoR array construction should succeed") } #[test] @@ -116,13 +114,18 @@ mod tests { reference, ); - let result = compare_constant(&lhs, 30i32, Nullability::NonNullable, CompareOperator::Eq) - .unwrap() - .unwrap(); + let result = compare_constant( + lhs.as_view(), + 30i32, + Nullability::NonNullable, + CompareOperator::Eq, + ) + .unwrap() + .unwrap(); assert_arrays_eq!(result, BoolArray::from_iter([false, true, false].map(Some))); let result = compare_constant( - &lhs, + lhs.as_view(), 12i32, Nullability::NonNullable, CompareOperator::NotEq, @@ -138,7 +141,7 @@ mod tests { CompareOperator::Gte, ] { assert!( - compare_constant(&lhs, 30i32, Nullability::NonNullable, op) + compare_constant(lhs.as_view(), 30i32, Nullability::NonNullable, op) .unwrap() .is_none() ); @@ -155,17 +158,27 @@ mod tests { ); assert_eq!( - compare_constant(&lhs, 30i32, Nullability::Nullable, CompareOperator::Eq) - .unwrap() - .unwrap() - .dtype(), + compare_constant( + lhs.as_view(), + 30i32, + Nullability::Nullable, + CompareOperator::Eq, + ) + .unwrap() + .unwrap() + .dtype(), &DType::Bool(Nullability::Nullable) ); assert_eq!( - compare_constant(&lhs, 30i32, Nullability::NonNullable, CompareOperator::Eq) - .unwrap() - .unwrap() - .dtype(), + compare_constant( + lhs.as_view(), + 30i32, + Nullability::NonNullable, + CompareOperator::Eq, + ) + .unwrap() + .unwrap() + .dtype(), &DType::Bool(Nullability::NonNullable) ); } @@ -179,16 +192,21 @@ mod tests { reference, ); - let result = compare_constant(&lhs, -1i32, Nullability::NonNullable, CompareOperator::Eq) - .unwrap() - .unwrap(); + let result = compare_constant( + lhs.as_view(), + -1i32, + Nullability::NonNullable, + CompareOperator::Eq, + ) + .unwrap() + .unwrap(); assert_arrays_eq!( result, BoolArray::from_iter([false, false, false].map(Some)) ); let result = compare_constant( - &lhs, + lhs.as_view(), -1i32, Nullability::NonNullable, CompareOperator::NotEq, @@ -212,7 +230,7 @@ mod tests { ); let result = compare_constant( - &lhs, + lhs.as_view(), 435090932899640449i64, Nullability::Nullable, CompareOperator::Eq, @@ -222,7 +240,7 @@ mod tests { assert_arrays_eq!(result, BoolArray::from_iter([Some(false), Some(true)])); let result = compare_constant( - &lhs, + lhs.as_view(), 435090932899640449i64, Nullability::Nullable, CompareOperator::NotEq, diff --git a/encodings/fastlanes/src/for/compute/is_sorted.rs b/encodings/fastlanes/src/for/compute/is_sorted.rs index 05d5e31836e..95504cdb7c7 100644 --- a/encodings/fastlanes/src/for/compute/is_sorted.rs +++ b/encodings/fastlanes/src/for/compute/is_sorted.rs @@ -10,6 +10,7 @@ use vortex_array::aggregate_fn::fns::is_sorted::IsSorted; use vortex_array::aggregate_fn::fns::is_sorted::is_sorted; use vortex_array::aggregate_fn::fns::is_sorted::is_strict_sorted; use vortex_array::aggregate_fn::kernels::DynAggregateKernel; +use vortex_array::arrays::PrimitiveArray; use vortex_array::scalar::Scalar; use vortex_error::VortexResult; @@ -34,9 +35,12 @@ impl DynAggregateKernel for FoRIsSortedKernel { }; let encoded = array.encoded().to_primitive(); - let unsigned_array = encoded - .reinterpret_cast(encoded.ptype().to_unsigned()) - .into_array(); + let unsigned_array = PrimitiveArray::from_buffer_handle( + encoded.buffer_handle().clone(), + encoded.ptype().to_unsigned(), + encoded.validity(), + ) + .into_array(); let result = if options.strict { is_strict_sorted(&unsigned_array, ctx)? diff --git a/encodings/fastlanes/src/for/compute/mod.rs b/encodings/fastlanes/src/for/compute/mod.rs index 7704dc02ff5..8bf7e93d029 100644 --- a/encodings/fastlanes/src/for/compute/mod.rs +++ b/encodings/fastlanes/src/for/compute/mod.rs @@ -16,7 +16,6 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::FoR; -use crate::FoRData; impl TakeExecute for FoR { fn take( @@ -25,7 +24,7 @@ impl TakeExecute for FoR { _ctx: &mut ExecutionCtx, ) -> VortexResult> { Ok(Some( - FoRData::try_new( + FoR::try_new( array.encoded().take(indices.clone())?, array.reference_scalar().clone(), )? @@ -36,7 +35,7 @@ impl TakeExecute for FoR { impl FilterReduce for FoR { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { - FoRData::try_new( + FoR::try_new( array.encoded().filter(mask.clone())?, array.reference_scalar().clone(), ) @@ -55,12 +54,11 @@ mod test { use vortex_buffer::buffer; use vortex_error::VortexExpect; + use crate::FoR; use crate::FoRArray; - use crate::FoRData; fn fa(encoded: ArrayRef, reference: Scalar) -> FoRArray { - FoRArray::try_from_data(FoRData::try_new(encoded, reference).unwrap()) - .vortex_expect("FoRData is always valid") + FoR::try_new(encoded, reference).vortex_expect("FoR array construction should succeed") } #[test] @@ -110,12 +108,11 @@ mod tests { use vortex_buffer::buffer; use vortex_error::VortexExpect; + use crate::FoR; use crate::FoRArray; - use crate::FoRData; fn fa(encoded: ArrayRef, reference: Scalar) -> FoRArray { - FoRArray::try_from_data(FoRData::try_new(encoded, reference).unwrap()) - .vortex_expect("FoRData is always valid") + FoR::try_new(encoded, reference).vortex_expect("FoR array construction should succeed") } #[rstest] diff --git a/encodings/fastlanes/src/for/vtable/mod.rs b/encodings/fastlanes/src/for/vtable/mod.rs index 80a2b165b91..157645c410f 100644 --- a/encodings/fastlanes/src/for/vtable/mod.rs +++ b/encodings/fastlanes/src/for/vtable/mod.rs @@ -8,6 +8,7 @@ use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; @@ -20,7 +21,6 @@ use vortex_array::dtype::DType; use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityVTableFromChild; @@ -48,29 +48,15 @@ vtable!(FoR, FoR, FoRData); impl VTable for FoR { type ArrayData = FoRData; - type Metadata = Scalar; - type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; - fn vtable(_array: &FoRData) -> &Self { - &FoR - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &FoRData) -> usize { - array.encoded().len() - } - - fn dtype(array: &FoRData) -> &DType { - array.reference_scalar().dtype() - } - - fn stats(array: &FoRData) -> &ArrayStats { - array.stats_set() + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate(dtype, len) } fn array_hash(array: &FoRData, state: &mut H, precision: Precision) { @@ -114,33 +100,27 @@ impl VTable for FoR { Ok(()) } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(array.reference_scalar().clone()) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { // Note that we **only** serialize the optional scalar value (not including the dtype). - Ok(Some(ScalarValue::to_proto_bytes(metadata.value()))) + Ok(Some(ScalarValue::to_proto_bytes( + array.reference_scalar().value(), + ))) } fn deserialize( - bytes: &[u8], - dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - session: &VortexSession, - ) -> VortexResult { - let scalar_value = ScalarValue::from_proto_bytes(bytes, dtype, session)?; - Scalar::try_new(dtype.clone(), scalar_value) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, - _buffers: &[BufferHandle], + metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + session: &VortexSession, ) -> VortexResult { + vortex_ensure!( + buffers.is_empty(), + "FoRArray expects 0 buffers, got {}", + buffers.len() + ); if children.len() != 1 { vortex_bail!( "Expected 1 child for FoR encoding, found {}", @@ -148,9 +128,11 @@ impl VTable for FoR { ) } + let scalar_value = ScalarValue::from_proto_bytes(metadata, dtype, session)?; + let reference = Scalar::try_new(dtype.clone(), scalar_value)?; let encoded = children.get(0, dtype, len)?; - FoRData::try_new(encoded, metadata.clone()) + FoRData::try_new(encoded, reference) } fn reduce_parent( @@ -183,7 +165,14 @@ impl FoR { /// Construct a new FoR array from an encoded array and a reference scalar. pub fn try_new(encoded: ArrayRef, reference: Scalar) -> VortexResult { - Array::try_from_data(FoRData::try_new(encoded, reference)?) + vortex_ensure!(!reference.is_null(), "Reference value cannot be null"); + let dtype = reference + .dtype() + .with_nullability(encoded.dtype().nullability()); + let reference = reference.cast(&dtype)?; + let len = encoded.len(); + let data = FoRData::try_new(encoded, reference)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(FoR, dtype, len, data)) }) } /// Encode a primitive array using Frame of Reference encoding. @@ -191,3 +180,16 @@ impl FoR { FoRData::encode(array) } } + +#[cfg(test)] +mod tests { + use vortex_array::scalar::ScalarValue; + use vortex_array::test_harness::check_metadata; + + #[cfg_attr(miri, ignore)] + #[test] + fn test_for_metadata() { + let metadata: Vec = ScalarValue::to_proto_bytes(Some(&ScalarValue::from(i64::MAX))); + check_metadata("for.metadata", &metadata); + } +} diff --git a/encodings/fastlanes/src/for/vtable/rules.rs b/encodings/fastlanes/src/for/vtable/rules.rs index 38d4b4186f2..3f839fe74dd 100644 --- a/encodings/fastlanes/src/for/vtable/rules.rs +++ b/encodings/fastlanes/src/for/vtable/rules.rs @@ -13,7 +13,6 @@ use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; use vortex_error::VortexResult; use crate::FoR; -use crate::FoRData; pub(super) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ // TODO: add BetweenReduceAdaptor(FoR) @@ -35,12 +34,12 @@ impl ArrayParentReduceRule for FoRFilterPushDownRule { parent: ArrayView<'_, Filter>, _child_idx: usize, ) -> VortexResult> { - let new_array = unsafe { - FoRData::new_unchecked( + Ok(Some( + FoR::try_new( child.encoded().filter(parent.filter_mask().clone())?, - child.reference.clone(), - ) - }; - Ok(Some(new_array.into_array())) + child.reference_scalar().clone(), + )? + .into_array(), + )) } } diff --git a/encodings/fastlanes/src/for/vtable/slice.rs b/encodings/fastlanes/src/for/vtable/slice.rs index c7571ac274a..acd8a4d0254 100644 --- a/encodings/fastlanes/src/for/vtable/slice.rs +++ b/encodings/fastlanes/src/for/vtable/slice.rs @@ -10,17 +10,15 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::FoR; -use crate::FoRData; impl SliceReduce for FoR { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - // SAFETY: Just slicing encoded data does not affect FOR. - Ok(Some(unsafe { - FoRData::new_unchecked( + Ok(Some( + FoR::try_new( array.encoded().slice(range)?, array.reference_scalar().clone(), - ) - .into_array() - })) + )? + .into_array(), + )) } } diff --git a/encodings/fastlanes/src/rle/array/mod.rs b/encodings/fastlanes/src/rle/array/mod.rs index d0d207c1795..2ee4a082dfa 100644 --- a/encodings/fastlanes/src/rle/array/mod.rs +++ b/encodings/fastlanes/src/rle/array/mod.rs @@ -3,8 +3,8 @@ use vortex_array::ArrayRef; use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; -use vortex_array::stats::ArrayStats; use vortex_error::VortexExpect as _; use vortex_error::VortexResult; use vortex_error::vortex_ensure; @@ -33,12 +33,9 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["values", "indices", "values_i #[derive(Clone, Debug)] pub struct RLEData { - pub(super) dtype: DType, pub(super) slots: Vec>, - pub(super) stats_set: ArrayStats, // Offset relative to the start of the chunk. pub(super) offset: usize, - pub(super) length: usize, } impl RLEData { @@ -47,6 +44,37 @@ impl RLEData { indices: &ArrayRef, value_idx_offsets: &ArrayRef, offset: usize, + length: usize, + ) -> VortexResult<()> { + Self::validate_parts(values, indices, value_idx_offsets, offset, length)?; + Ok(()) + } + + pub(crate) fn validate_against_outer(&self, dtype: &DType, length: usize) -> VortexResult<()> { + Self::validate_parts( + self.values(), + self.indices(), + self.values_idx_offsets(), + self.offset, + length, + )?; + let expected_dtype = DType::Primitive( + self.values().dtype().as_ptype(), + self.indices().dtype().nullability(), + ); + vortex_ensure!( + dtype == &expected_dtype, + "RLE dtype mismatch: expected {expected_dtype}, got {dtype}" + ); + Ok(()) + } + + fn validate_parts( + values: &ArrayRef, + indices: &ArrayRef, + value_idx_offsets: &ArrayRef, + offset: usize, + length: usize, ) -> VortexResult<()> { vortex_ensure!( offset < 1024, @@ -55,8 +83,11 @@ impl RLEData { ); vortex_ensure!( - values.dtype().is_primitive(), - "RLE values must be a primitive type, got {}", + matches!( + values.dtype(), + DType::Primitive(_, Nullability::NonNullable) + ), + "RLE values must be a non-nullable primitive type, got {}", values.dtype() ); @@ -71,6 +102,16 @@ impl RLEData { "RLE value idx offsets must be non-nullable unsigned integer, got {}", value_idx_offsets.dtype() ); + vortex_ensure!( + indices.len().is_multiple_of(FL_CHUNK_SIZE), + "RLE indices length must be a multiple of {FL_CHUNK_SIZE}, got {}", + indices.len() + ); + vortex_ensure!( + offset + length <= indices.len(), + "RLE offset + length, {offset} + {length}, must not exceed the indices length {}", + indices.len() + ); vortex_ensure!( indices.len().div_ceil(FL_CHUNK_SIZE) == value_idx_offsets.len(), @@ -104,18 +145,11 @@ impl RLEData { offset: usize, length: usize, ) -> VortexResult { - assert_eq!(indices.len() % FL_CHUNK_SIZE, 0); - Self::validate(&values, &indices, &values_idx_offsets, offset)?; - - // Ensure that the DType has the same nullability as the indices array. - let dtype = DType::Primitive(values.dtype().as_ptype(), indices.dtype().nullability()); + Self::validate(&values, &indices, &values_idx_offsets, offset, length)?; Ok(Self { - dtype, slots: vec![Some(values), Some(indices), Some(values_idx_offsets)], - stats_set: ArrayStats::default(), offset, - length, }) } @@ -124,42 +158,20 @@ impl RLEData { /// # Safety /// The caller must ensure that: /// - `offset + length` does not exceed the length of the indices array - /// - The `dtype` is consistent with the values array's primitive type and validity nullability /// - The `indices` array contains valid indices into chunks of the `values` array /// - The `values_idx_offsets` array contains valid chunk start offsets - /// - The `validity` array has the same length as `length` pub unsafe fn new_unchecked( values: ArrayRef, indices: ArrayRef, values_idx_offsets: ArrayRef, - dtype: DType, offset: usize, - length: usize, ) -> Self { Self { - dtype, slots: vec![Some(values), Some(indices), Some(values_idx_offsets)], - stats_set: ArrayStats::default(), offset, - length, } } - #[inline] - pub fn len(&self) -> usize { - self.length - } - - #[inline] - pub fn is_empty(&self) -> bool { - self.length == 0 - } - - #[inline] - pub fn dtype(&self) -> &DType { - &self.dtype - } - #[inline] pub fn values(&self) -> &ArrayRef { self.slots[VALUES_SLOT] @@ -211,10 +223,6 @@ impl RLEData { pub fn offset(&self) -> usize { self.offset } - - pub(crate) fn stats_set(&self) -> &ArrayStats { - &self.stats_set - } } #[cfg(test)] @@ -229,8 +237,8 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::serde::ArrayParts; use vortex_array::serde::SerializeOptions; + use vortex_array::serde::SerializedArray; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::ByteBufferMut; @@ -239,7 +247,7 @@ mod tests { use vortex_session::registry::ReadContext; use crate::FL_CHUNK_SIZE; - use crate::RLEArray; + use crate::RLE; use crate::RLEData; use crate::test::SESSION; @@ -252,10 +260,8 @@ mod tests { PrimitiveArray::from_iter([0u16, 0, 1, 1, 2].iter().cycle().take(1024).copied()) .into_array(); let values_idx_offsets = PrimitiveArray::from_iter([0u64]).into_array(); - let rle_array = RLEArray::try_from_data( - RLEData::try_new(values, indices, values_idx_offsets, 0, 5).unwrap(), - ) - .vortex_expect("RLEData is always valid"); + let rle_array = RLE::try_new(values, indices, values_idx_offsets, 0, 5) + .vortex_expect("RLEData is always valid"); assert_eq!(rle_array.len(), 5); assert_eq!(rle_array.values().len(), 3); @@ -282,10 +288,8 @@ mod tests { ) .into_array(); - let rle_array = RLEArray::try_from_data( - RLEData::try_new(values, indices_with_validity, values_idx_offsets, 0, 3).unwrap(), - ) - .vortex_expect("RLEData is always valid"); + let rle_array = RLE::try_new(values, indices_with_validity, values_idx_offsets, 0, 3) + .vortex_expect("RLEData is always valid"); assert_eq!(rle_array.len(), 3); assert_eq!(rle_array.values().len(), 2); @@ -314,10 +318,8 @@ mod tests { ) .into_array(); - let rle_array = RLEArray::try_from_data( - RLEData::try_new(values, indices_with_validity, values_idx_offsets, 0, 5).unwrap(), - ) - .vortex_expect("RLEData is always valid"); + let rle_array = RLE::try_new(values, indices_with_validity, values_idx_offsets, 0, 5) + .vortex_expect("RLEData is always valid"); let valid_slice = rle_array.slice(0..3).unwrap().to_primitive(); // TODO(joe): replace with compute null count @@ -347,10 +349,8 @@ mod tests { ) .into_array(); - let rle_array = RLEArray::try_from_data( - RLEData::try_new(values, indices_with_validity, values_idx_offsets, 0, 5).unwrap(), - ) - .vortex_expect("RLEData is always valid"); + let rle_array = RLE::try_new(values, indices_with_validity, values_idx_offsets, 0, 5) + .vortex_expect("RLEData is always valid"); // TODO(joe): replace with compute null count let invalid_slice = rle_array @@ -385,10 +385,8 @@ mod tests { ) .into_array(); - let rle_array = RLEArray::try_from_data( - RLEData::try_new(values, indices_with_validity, values_idx_offsets, 0, 4).unwrap(), - ) - .vortex_expect("RLEData is always valid"); + let rle_array = RLE::try_new(values, indices_with_validity, values_idx_offsets, 0, 4) + .vortex_expect("RLEData is always valid"); let sliced_array = rle_array.slice(1..4).unwrap(); let validity_mask = sliced_array.validity_mask().unwrap(); @@ -408,15 +406,12 @@ mod tests { let values = PrimitiveArray::from_iter(Vec::::new()).into_array(); let indices = PrimitiveArray::from_iter(Vec::::new()).into_array(); let values_idx_offsets = PrimitiveArray::from_iter(Vec::::new()).into_array(); - let rle_array = RLEArray::try_from_data( - RLEData::try_new( - values, - indices.clone(), - values_idx_offsets, - 0, - indices.len(), - ) - .unwrap(), + let rle_array = RLE::try_new( + values, + indices.clone(), + values_idx_offsets, + 0, + indices.len(), ) .vortex_expect("RLEData is always valid"); @@ -429,10 +424,8 @@ mod tests { let values = PrimitiveArray::from_iter([10u32, 20, 30, 40]).into_array(); let indices = PrimitiveArray::from_iter([0u16, 1].repeat(1024)).into_array(); let values_idx_offsets = PrimitiveArray::from_iter([0u64, 2]).into_array(); - let rle_array = RLEArray::try_from_data( - RLEData::try_new(values, indices, values_idx_offsets, 0, 2048).unwrap(), - ) - .vortex_expect("RLEData is always valid"); + let rle_array = RLE::try_new(values, indices, values_idx_offsets, 0, 2048) + .vortex_expect("RLEData is always valid"); assert_eq!(rle_array.len(), 2048); assert_eq!(rle_array.values().len(), 4); @@ -461,7 +454,7 @@ mod tests { } let concat = concat.freeze(); - let parts = ArrayParts::try_from(concat).unwrap(); + let parts = SerializedArray::try_from(concat).unwrap(); let decoded = parts .decode( &DType::Primitive(PType::U32, Nullability::NonNullable), @@ -481,15 +474,12 @@ mod tests { let primitive = PrimitiveArray::from_iter((0..2048).map(|i| (i / 100) as u32)); let rle_array = RLEData::encode(&primitive).unwrap(); - let sliced = RLEArray::try_from_data( - RLEData::try_new( - rle_array.values().clone(), - rle_array.indices().clone(), - rle_array.values_idx_offsets().clone(), - 100, - 100, - ) - .unwrap(), + let sliced = RLE::try_new( + rle_array.values().clone(), + rle_array.indices().clone(), + rle_array.values_idx_offsets().clone(), + 100, + 100, ) .vortex_expect("RLEData is always valid"); assert_eq!(sliced.len(), 100); @@ -507,7 +497,7 @@ mod tests { } let concat = concat.freeze(); - let parts = ArrayParts::try_from(concat).unwrap(); + let parts = SerializedArray::try_from(concat).unwrap(); let decoded = parts .decode( sliced.dtype(), @@ -550,16 +540,15 @@ mod tests { // Reconstruct the outer RLE with re-encoded indices. // SAFETY: we only replace the indices child; all other invariants hold. - let reconstructed = RLEArray::try_from_data(unsafe { - RLEData::new_unchecked( + let reconstructed = unsafe { + RLE::new_unchecked( rle.values().clone(), re_encoded.into_array(), rle.values_idx_offsets().clone(), - rle.dtype().clone(), rle.offset(), rle.len(), ) - })?; + }; // Decompress — panicked before the fill_forward_nulls chunk-boundary fix. let decoded = reconstructed.as_array().to_primitive(); diff --git a/encodings/fastlanes/src/rle/array/rle_compress.rs b/encodings/fastlanes/src/rle/array/rle_compress.rs index a432097b04e..11270da5486 100644 --- a/encodings/fastlanes/src/rle/array/rle_compress.rs +++ b/encodings/fastlanes/src/rle/array/rle_compress.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use arrayref::array_mut_ref; -use fastlanes::RLE; +use fastlanes::RLE as FastLanesRLE; use vortex_array::IntoArray; use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; @@ -15,6 +15,7 @@ use vortex_buffer::BufferMut; use vortex_error::VortexResult; use crate::FL_CHUNK_SIZE; +use crate::RLE; use crate::RLEArray; use crate::RLEData; use crate::fill_forward_nulls; @@ -31,8 +32,8 @@ impl RLEData { /// In case the input array length is % 1024 != 0, the last chunk is padded. fn rle_encode_typed(array: &PrimitiveArray) -> VortexResult where - T: NativePType + RLE, - NativeValue: RLE, + T: NativePType + FastLanesRLE, + NativeValue: FastLanesRLE, { // Fill-forward null values so the RLE encoder doesn't see garbage at null positions, // which would create spurious run boundaries and inflate the dictionary. @@ -98,13 +99,13 @@ where // SAFETY: NativeValue is repr(transparent) to T. let values_buf = unsafe { values_buf.transmute::().freeze() }; - RLEArray::try_from_data(RLEData::try_new( + RLE::try_new( values_buf.into_array(), PrimitiveArray::new(indices_buf.freeze(), padded_validity(array)).into_array(), values_idx_offsets.into_array(), 0, array.len(), - )?) + ) } /// Returns validity padded to the next 1024 chunk for a given array. @@ -264,7 +265,7 @@ mod tests { #[case(vec![f16::ZERO, f16::NEG_ZERO])] #[case(vec![0f32, -0f32])] #[case(vec![0f64, -0f64])] - fn test_float_zeros(#[case] values: Vec) { + fn test_float_zeros(#[case] values: Vec) { let primitive = PrimitiveArray::from_iter(values); let rle = RLEData::encode(&primitive).unwrap(); let decoded = rle.as_array().to_primitive(); diff --git a/encodings/fastlanes/src/rle/compute/cast.rs b/encodings/fastlanes/src/rle/compute/cast.rs index 88078862816..c3ae7d42e81 100644 --- a/encodings/fastlanes/src/rle/compute/cast.rs +++ b/encodings/fastlanes/src/rle/compute/cast.rs @@ -3,17 +3,20 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; +use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; -use crate::RLEData; use crate::rle::RLE; impl CastReduce for RLE { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // Cast RLE values. - let casted_values = array.values().cast(dtype.clone())?; + let casted_values = array + .values() + .cast(DType::Primitive(dtype.as_ptype(), Nullability::NonNullable))?; // Cast RLE indices such that validity matches the target dtype. let casted_indices = if array.indices().dtype().nullability() != dtype.nullability() { @@ -25,17 +28,16 @@ impl CastReduce for RLE { array.indices().clone() }; - Ok(Some(unsafe { - RLEData::new_unchecked( + Ok(Some( + RLE::try_new( casted_values, casted_indices, array.values_idx_offsets().clone(), - dtype.clone(), array.offset(), array.len(), - ) - .into() - })) + )? + .into_array(), + )) } } diff --git a/encodings/fastlanes/src/rle/kernel.rs b/encodings/fastlanes/src/rle/kernel.rs index 1da88206913..65f8bfdb907 100644 --- a/encodings/fastlanes/src/rle/kernel.rs +++ b/encodings/fastlanes/src/rle/kernel.rs @@ -14,7 +14,6 @@ use vortex_error::VortexResult; use crate::FL_CHUNK_SIZE; use crate::RLE; -use crate::RLEData; pub(crate) static PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ParentKernelSet::lift(&SliceExecuteAdaptor(RLE))]); @@ -46,18 +45,16 @@ impl SliceKernel for RLE { .indices() .slice(chunk_start_idx * FL_CHUNK_SIZE..chunk_end_idx * FL_CHUNK_SIZE)?; - // SAFETY: Slicing preserves all invariants. - Ok(Some(unsafe { - RLEData::new_unchecked( + Ok(Some( + RLE::try_new( sliced_values, sliced_indices, sliced_values_idx_offsets, - array.dtype().clone(), // Keep the offset relative to the first chunk. (array.offset() + range.start) % FL_CHUNK_SIZE, range.len(), - ) - .into_array() - })) + )? + .into_array(), + )) } } diff --git a/encodings/fastlanes/src/rle/vtable/mod.rs b/encodings/fastlanes/src/rle/vtable/mod.rs index 63eeb7b5706..776fa1b84ad 100644 --- a/encodings/fastlanes/src/rle/vtable/mod.rs +++ b/encodings/fastlanes/src/rle/vtable/mod.rs @@ -8,24 +8,21 @@ use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::VTable; -use vortex_array::vtable::ValidityVTableFromChildSliceHelper; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; @@ -61,29 +58,15 @@ pub struct RLEMetadata { impl VTable for RLE { type ArrayData = RLEData; - type Metadata = ProstMetadata; - type OperationsVTable = Self; - type ValidityVTable = ValidityVTableFromChildSliceHelper; - - fn vtable(_array: &RLEData) -> &Self { - &RLE - } + type ValidityVTable = Self; fn id(&self) -> ArrayId { Self::ID } - fn len(array: &RLEData) -> usize { - array.len() - } - - fn dtype(array: &RLEData) -> &DType { - array.dtype() - } - - fn stats(array: &RLEData) -> &ArrayStats { - array.stats_set() + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate_against_outer(dtype, len) } fn array_hash(array: &RLEData, state: &mut H, precision: Precision) { @@ -141,39 +124,36 @@ impl VTable for RLE { Ok(()) } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(RLEMetadata { - values_len: array.values().len() as u64, - indices_len: array.indices().len() as u64, - indices_ptype: PType::try_from(array.indices().dtype())? as i32, - values_idx_offsets_len: array.values_idx_offsets().len() as u64, - values_idx_offsets_ptype: PType::try_from(array.values_idx_offsets().dtype())? as i32, - offset: array.offset() as u64, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.0.encode_to_vec())) + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some( + RLEMetadata { + values_len: array.values().len() as u64, + indices_len: array.indices().len() as u64, + indices_ptype: PType::try_from(array.indices().dtype())? as i32, + values_idx_offsets_len: array.values_idx_offsets().len() as u64, + values_idx_offsets_ptype: PType::try_from(array.values_idx_offsets().dtype())? + as i32, + offset: array.offset() as u64, + } + .encode_to_vec(), + )) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata(RLEMetadata::decode(bytes)?)) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, - _buffers: &[BufferHandle], + metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { - let metadata = &metadata.0; + vortex_ensure!( + buffers.is_empty(), + "RLEArray expects 0 buffers, got {}", + buffers.len() + ); + let metadata = RLEMetadata::decode(metadata)?; let values = children.get( 0, &DType::Primitive(dtype.as_ptype(), Nullability::NonNullable), @@ -226,6 +206,18 @@ pub struct RLE; impl RLE { pub const ID: ArrayId = ArrayId::new_ref("fastlanes.rle"); + pub fn try_new( + values: ArrayRef, + indices: ArrayRef, + values_idx_offsets: ArrayRef, + offset: usize, + length: usize, + ) -> VortexResult { + let dtype = DType::Primitive(values.dtype().as_ptype(), indices.dtype().nullability()); + let data = RLEData::try_new(values, indices, values_idx_offsets, offset, length)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(RLE, dtype, length, data)) }) + } + /// Create a new RLE array without validation. /// /// # Safety @@ -234,14 +226,12 @@ impl RLE { values: ArrayRef, indices: ArrayRef, values_idx_offsets: ArrayRef, - dtype: DType, offset: usize, length: usize, ) -> RLEArray { - Array::try_from_data(unsafe { - RLEData::new_unchecked(values, indices, values_idx_offsets, dtype, offset, length) - }) - .vortex_expect("RLEData is always valid") + let dtype = DType::Primitive(values.dtype().as_ptype(), indices.dtype().nullability()); + let data = unsafe { RLEData::new_unchecked(values, indices, values_idx_offsets, offset) }; + unsafe { Array::from_parts_unchecked(ArrayParts::new(RLE, dtype, length, data)) } } /// Encode a primitive array using FastLanes RLE. @@ -252,9 +242,9 @@ impl RLE { #[cfg(test)] mod tests { + use prost::Message; use vortex_array::test_harness::check_metadata; - use super::ProstMetadata; use super::RLEMetadata; #[cfg_attr(miri, ignore)] @@ -262,14 +252,15 @@ mod tests { fn test_rle_metadata() { check_metadata( "rle.metadata", - ProstMetadata(RLEMetadata { + &RLEMetadata { values_len: u64::MAX, indices_len: u64::MAX, indices_ptype: i32::MAX, values_idx_offsets_len: u64::MAX, values_idx_offsets_ptype: i32::MAX, offset: u64::MAX, - }), + } + .encode_to_vec(), ); } } diff --git a/encodings/fastlanes/src/rle/vtable/operations.rs b/encodings/fastlanes/src/rle/vtable/operations.rs index 8804d30d178..e8dfc159f70 100644 --- a/encodings/fastlanes/src/rle/vtable/operations.rs +++ b/encodings/fastlanes/src/rle/vtable/operations.rs @@ -47,6 +47,7 @@ mod tests { use vortex_buffer::buffer; use super::*; + use crate::RLE; use crate::RLEArray; use crate::RLEData; @@ -65,15 +66,12 @@ mod tests { .into_array(); let values_idx_offsets = PrimitiveArray::from_iter([0u64]).into_array(); - RLEArray::try_from_data( - RLEData::try_new( - values, - indices.clone(), - values_idx_offsets, - 0, - indices.len(), - ) - .unwrap(), + RLE::try_new( + values, + indices.clone(), + values_idx_offsets, + 0, + indices.len(), ) .vortex_expect("RLEData is always valid") } @@ -103,15 +101,12 @@ mod tests { ) .into_array(); - RLEArray::try_from_data( - RLEData::try_new( - values, - indices.clone(), - values_idx_offsets, - 0, - indices.len(), - ) - .unwrap(), + RLE::try_new( + values, + indices.clone(), + values_idx_offsets, + 0, + indices.len(), ) .vortex_expect("RLEData is always valid") } diff --git a/encodings/fastlanes/src/rle/vtable/validity.rs b/encodings/fastlanes/src/rle/vtable/validity.rs index f46998774cd..ecef8047cb1 100644 --- a/encodings/fastlanes/src/rle/vtable/validity.rs +++ b/encodings/fastlanes/src/rle/vtable/validity.rs @@ -1,22 +1,16 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use vortex_array::ArrayRef; -use vortex_array::vtable::ValidityChild; -use vortex_array::vtable::ValidityChildSliceHelper; +use vortex_array::ArrayView; +use vortex_array::vtable::ValidityVTable; +use vortex_error::VortexResult; -use super::RLE; -use crate::RLEData; +use crate::rle::RLE; -impl ValidityChild for RLE { - fn validity_child(array: &RLEData) -> &ArrayRef { - array.indices() - } -} - -impl ValidityChildSliceHelper for RLEData { - fn unsliced_child_and_slice(&self) -> (&ArrayRef, usize, usize) { - let (start, len) = (self.offset(), self.len()); - (self.indices(), start, start + len) +impl ValidityVTable for RLE { + fn validity(array: ArrayView<'_, RLE>) -> VortexResult { + let start = array.offset(); + let stop = start + array.len(); + array.indices().slice(start..stop)?.validity() } } diff --git a/encodings/fsst/public-api.lock b/encodings/fsst/public-api.lock index 32522dd3f04..4989f13dca7 100644 --- a/encodings/fsst/public-api.lock +++ b/encodings/fsst/public-api.lock @@ -20,8 +20,6 @@ impl vortex_array::array::vtable::VTable for vortex_fsst::FSST pub type vortex_fsst::FSST::ArrayData = vortex_fsst::FSSTData -pub type vortex_fsst::FSST::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_fsst::FSST::OperationsVTable = vortex_fsst::FSST pub type vortex_fsst::FSST::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -36,11 +34,7 @@ pub fn vortex_fsst::FSST::buffer(array: vortex_array::array::view::ArrayView<'_, pub fn vortex_fsst::FSST::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_fsst::FSST::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_fsst::FSST::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_fsst::FSST::dtype(array: &vortex_fsst::FSSTData) -> &vortex_array::dtype::DType +pub fn vortex_fsst::FSST::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_fsst::FSST::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -48,23 +42,17 @@ pub fn vortex_fsst::FSST::execute_parent(array: vortex_array::array::view::Array pub fn vortex_fsst::FSST::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_fsst::FSST::len(array: &vortex_fsst::FSSTData) -> usize - -pub fn vortex_fsst::FSST::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_fsst::FSST::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_fsst::FSST::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_fsst::FSST::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_fsst::FSST::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_fsst::FSST::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_fsst::FSST::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_fsst::FSST::stats(array: &vortex_fsst::FSSTData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_fsst::FSST::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_fsst::FSST::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_fsst::FSST::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -112,8 +100,6 @@ pub fn vortex_fsst::FSSTData::compressor(&self) -> &fsst::Compressor pub fn vortex_fsst::FSSTData::decompressor(&self) -> fsst::Decompressor<'_> -pub fn vortex_fsst::FSSTData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_fsst::FSSTData::is_empty(&self) -> bool pub fn vortex_fsst::FSSTData::len(&self) -> usize @@ -122,28 +108,22 @@ pub fn vortex_fsst::FSSTData::symbol_lengths(&self) -> &vortex_buffer::buffer::B pub fn vortex_fsst::FSSTData::symbols(&self) -> &vortex_buffer::buffer::Buffer -pub fn vortex_fsst::FSSTData::try_new(dtype: vortex_array::dtype::DType, symbols: vortex_buffer::buffer::Buffer, symbol_lengths: vortex_buffer::buffer::Buffer, codes: vortex_array::arrays::varbin::vtable::VarBinArray, uncompressed_lengths: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult +pub fn vortex_fsst::FSSTData::try_new(symbols: vortex_buffer::buffer::Buffer, symbol_lengths: vortex_buffer::buffer::Buffer, codes: vortex_array::arrays::varbin::vtable::VarBinArray, uncompressed_lengths: vortex_array::array::erased::ArrayRef, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult pub fn vortex_fsst::FSSTData::uncompressed_lengths(&self) -> &vortex_array::array::erased::ArrayRef pub fn vortex_fsst::FSSTData::uncompressed_lengths_dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_fsst::FSSTData::validate(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> + impl core::clone::Clone for vortex_fsst::FSSTData pub fn vortex_fsst::FSSTData::clone(&self) -> vortex_fsst::FSSTData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_fsst::FSSTData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_fsst::FSSTData pub fn vortex_fsst::FSSTData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_fsst::FSSTData - -pub fn vortex_fsst::FSSTData::into_array(self) -> vortex_array::array::erased::ArrayRef - pub struct vortex_fsst::FSSTMetadata impl vortex_fsst::FSSTMetadata diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index b98d5d9edd6..d716bc7608b 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -9,20 +9,19 @@ use std::sync::LazyLock; use fsst::Compressor; use fsst::Decompressor; use fsst::Symbol; +use prost::Message as _; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::Canonical; -use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; -use vortex_array::SerializeMetadata; use vortex_array::arrays::VarBin; use vortex_array::arrays::VarBinArray; use vortex_array::buffer::BufferHandle; @@ -32,7 +31,6 @@ use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; @@ -74,29 +72,15 @@ impl FSSTMetadata { impl VTable for FSST { type ArrayData = FSSTData; - - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; - fn vtable(_array: &Self::ArrayData) -> &Self { - &FSST - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &FSSTData) -> usize { - array.codes().len() - } - - fn dtype(array: &FSSTData) -> &DType { - &array.dtype - } - - fn stats(array: &FSSTData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate(dtype, len) } fn array_hash(array: &FSSTData, state: &mut H, precision: Precision) { @@ -147,60 +131,26 @@ impl VTable for FSST { } } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(FSSTMetadata { - uncompressed_lengths_ptype: array.uncompressed_lengths().dtype().as_ptype().into(), - codes_offsets_ptype: array.codes.offsets().dtype().as_ptype().into(), - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) - } - - fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata( - as DeserializeMetadata>::deserialize(bytes)?, + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some( + FSSTMetadata { + uncompressed_lengths_ptype: array.uncompressed_lengths().dtype().as_ptype().into(), + codes_offsets_ptype: array.codes.offsets().dtype().as_ptype().into(), + } + .encode_to_vec(), )) } - fn append_to_builder( - array: ArrayView<'_, Self>, - builder: &mut dyn ArrayBuilder, - ctx: &mut ExecutionCtx, - ) -> VortexResult<()> { - let Some(builder) = builder.as_any_mut().downcast_mut::() else { - builder.extend_from_array( - &array - .array() - .clone() - .execute::(ctx)? - .into_array(), - ); - return Ok(()); - }; - - // Decompress the whole block of data into a new buffer, and create some views - // from it instead. - let (buffers, views) = fsst_decode_views(&array, builder.completed_block_count(), ctx)?; - - builder.push_buffer_and_adjusted_views(&buffers, &views, array.array().validity_mask()?); - Ok(()) - } - - fn build( + fn deserialize( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = FSSTMetadata::decode(metadata)?; let symbols = Buffer::::from_byte_buffer(buffers[0].clone().try_to_host_sync()?); let symbol_lengths = Buffer::::from_byte_buffer(buffers[1].clone().try_to_host_sync()?); @@ -222,19 +172,13 @@ impl VTable for FSST { let uncompressed_lengths = children.get( 1, &DType::Primitive( - metadata.0.get_uncompressed_lengths_ptype()?, + metadata.get_uncompressed_lengths_ptype()?, Nullability::NonNullable, ), len, )?; - return FSSTData::try_new( - dtype.clone(), - symbols, - symbol_lengths, - codes, - uncompressed_lengths, - ); + return FSSTData::try_new(symbols, symbol_lengths, codes, uncompressed_lengths, dtype); } // Check for the current deserialization path. @@ -242,7 +186,7 @@ impl VTable for FSST { let uncompressed_lengths = children.get( 0, &DType::Primitive( - metadata.0.get_uncompressed_lengths_ptype()?, + metadata.get_uncompressed_lengths_ptype()?, Nullability::NonNullable, ), len, @@ -275,13 +219,7 @@ impl VTable for FSST { codes_validity, )?; - return FSSTData::try_new( - dtype.clone(), - symbols, - symbol_lengths, - codes, - uncompressed_lengths, - ); + return FSSTData::try_new(symbols, symbol_lengths, codes, uncompressed_lengths, dtype); } vortex_bail!( @@ -312,7 +250,7 @@ impl VTable for FSST { .vortex_expect("FSSTArray requires codes_offsets slot"); let codes_validity = match &slots[CODES_VALIDITY_SLOT] { Some(v) => Validity::Array(v.clone()), - None => Validity::from(array.dtype.nullability()), + None => Validity::from(array.codes.dtype().nullability()), }; let codes = VarBinArray::try_new( codes_offsets, @@ -328,7 +266,31 @@ impl VTable for FSST { } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { - canonicalize_fsst(&array, ctx).map(ExecutionResult::done) + canonicalize_fsst(array.as_view(), ctx).map(ExecutionResult::done) + } + + fn append_to_builder( + array: ArrayView<'_, Self>, + builder: &mut dyn ArrayBuilder, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let Some(builder) = builder.as_any_mut().downcast_mut::() else { + builder.extend_from_array( + &array + .array() + .clone() + .execute::(ctx)? + .into_array(), + ); + return Ok(()); + }; + + // Decompress the whole block of data into a new buffer, and create some views + // from it instead. + let (buffers, views) = fsst_decode_views(&array, builder.completed_block_count(), ctx)?; + + builder.push_buffer_and_adjusted_views(&buffers, &views, array.array().validity_mask()?); + Ok(()) } fn execute_parent( @@ -361,7 +323,6 @@ pub(crate) const SLOT_NAMES: [&str; NUM_SLOTS] = #[derive(Clone)] pub struct FSSTData { - dtype: DType, symbols: Buffer, symbol_lengths: Buffer, codes: VarBinArray, @@ -369,7 +330,6 @@ pub struct FSSTData { codes_array: ArrayRef, /// Lengths of the original values before compression, can be compressed. slots: Vec>, - stats_set: ArrayStats, /// Memoized compressor used for push-down of compute by compressing the RHS. compressor: Arc Compressor + Send>>>, @@ -378,7 +338,6 @@ pub struct FSSTData { impl Debug for FSSTData { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("FSSTArray") - .field("dtype", &self.dtype) .field("symbols", &self.symbols) .field("symbol_lengths", &self.symbol_lengths) .field("codes", &self.codes) @@ -401,13 +360,23 @@ impl FSST { codes: VarBinArray, uncompressed_lengths: ArrayRef, ) -> VortexResult { - Array::try_from_data(FSSTData::try_new( - dtype, - symbols, - symbol_lengths, - codes, - uncompressed_lengths, - )?) + let len = codes.len(); + let data = FSSTData::try_new(symbols, symbol_lengths, codes, uncompressed_lengths, &dtype)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(FSST, dtype, len, data)) }) + } + + pub(crate) unsafe fn new_unchecked( + dtype: DType, + symbols: Buffer, + symbol_lengths: Buffer, + codes: VarBinArray, + uncompressed_lengths: ArrayRef, + ) -> FSSTArray { + let len = codes.len(); + let data = unsafe { + FSSTData::new_unchecked(symbols, symbol_lengths, codes, uncompressed_lengths) + }; + unsafe { Array::from_parts_unchecked(ArrayParts::new(FSST, dtype, len, data)) } } } @@ -421,12 +390,56 @@ impl FSSTData { /// Each code corresponds either to a symbol, or to the "escape code", /// which tells the decoder to emit the following byte without doing a table lookup. pub fn try_new( - dtype: DType, symbols: Buffer, symbol_lengths: Buffer, codes: VarBinArray, uncompressed_lengths: ArrayRef, + dtype: &DType, ) -> VortexResult { + Self::validate_parts( + &symbols, + &symbol_lengths, + &codes, + &uncompressed_lengths, + dtype, + codes.len(), + )?; + + // SAFETY: all components validated above + unsafe { + Ok(Self::new_unchecked( + symbols, + symbol_lengths, + codes, + uncompressed_lengths, + )) + } + } + + pub fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { + Self::validate_parts( + &self.symbols, + &self.symbol_lengths, + &self.codes, + self.uncompressed_lengths(), + dtype, + len, + ) + } + + fn validate_parts( + symbols: &Buffer, + symbol_lengths: &Buffer, + codes: &VarBinArray, + uncompressed_lengths: &ArrayRef, + dtype: &DType, + len: usize, + ) -> VortexResult<()> { + vortex_ensure!( + matches!(dtype, DType::Binary(_) | DType::Utf8(_)), + "FSST arrays must be Binary or Utf8, found {dtype}" + ); + // Check: symbols must not have length > MAX_CODE if symbols.len() > 255 { vortex_bail!(InvalidArgument: "symbols array must have length <= 255"); @@ -435,7 +448,11 @@ impl FSSTData { vortex_bail!(InvalidArgument: "symbols and symbol_lengths arrays must have same length"); } - if uncompressed_lengths.len() != codes.len() { + if codes.len() != len { + vortex_bail!(InvalidArgument: "codes must have same len as outer array"); + } + + if uncompressed_lengths.len() != len { vortex_bail!(InvalidArgument: "uncompressed_lengths must be same len as codes"); } @@ -447,21 +464,14 @@ impl FSSTData { if !matches!(codes.dtype(), DType::Binary(_)) { vortex_bail!(InvalidArgument: "codes array must be DType::Binary type"); } - - // SAFETY: all components validated above - unsafe { - Ok(Self::new_unchecked( - dtype, - symbols, - symbol_lengths, - codes, - uncompressed_lengths, - )) + if codes.dtype().nullability() != dtype.nullability() { + vortex_bail!(InvalidArgument: "codes nullability must match outer dtype nullability"); } + + Ok(()) } pub(crate) unsafe fn new_unchecked( - dtype: DType, symbols: Buffer, symbol_lengths: Buffer, codes: VarBinArray, @@ -478,7 +488,6 @@ impl FSSTData { let codes_validity_slot = validity_to_child(&codes.validity(), codes.len()); Self { - dtype, symbols, symbol_lengths, codes, @@ -488,7 +497,6 @@ impl FSSTData { codes_offsets_slot, codes_validity_slot, ], - stats_set: Default::default(), compressor, } } @@ -503,11 +511,6 @@ impl FSSTData { self.codes.len() == 0 } - /// Returns the logical data type of the array. - pub fn dtype(&self) -> &DType { - &self.dtype - } - /// Access the symbol table array pub fn symbols(&self) -> &Buffer { &self.symbols @@ -564,9 +567,10 @@ impl ValidityChild for FSST { mod test { use fsst::Compressor; use fsst::Symbol; + use prost::Message; + use vortex_array::DynVTable; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; - use vortex_array::ProstMetadata; use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::VarBinViewArray; @@ -575,7 +579,6 @@ mod test { use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::test_harness::check_metadata; - use vortex_array::vtable::VTable; use vortex_buffer::Buffer; use vortex_error::VortexError; @@ -588,10 +591,11 @@ mod test { fn test_fsst_metadata() { check_metadata( "fsst.metadata", - ProstMetadata(FSSTMetadata { + &FSSTMetadata { uncompressed_lengths_ptype: PType::U64 as i32, codes_offsets_ptype: PType::I32 as i32, - }), + } + .encode_to_vec(), ); } @@ -636,10 +640,12 @@ mod test { fsst_array.uncompressed_lengths().clone(), ]; - let fsst = FSST::build( + let fsst = DynVTable::build( + &FSST, + FSST::ID, &DType::Utf8(Nullability::NonNullable), 2, - &ProstMetadata(FSSTMetadata { + &FSSTMetadata { uncompressed_lengths_ptype: fsst_array .uncompressed_lengths() .dtype() @@ -647,9 +653,11 @@ mod test { .into(), // Legacy array did not store this field, use Protobuf default of 0. codes_offsets_ptype: 0, - }), + } + .encode_to_vec(), &buffers, &children.as_slice(), + &LEGACY_SESSION, ) .unwrap(); diff --git a/encodings/fsst/src/canonical.rs b/encodings/fsst/src/canonical.rs index 41df50f913b..64fb8ea2117 100644 --- a/encodings/fsst/src/canonical.rs +++ b/encodings/fsst/src/canonical.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use vortex_array::ArrayRef; +use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::PrimitiveArray; @@ -17,13 +18,14 @@ use vortex_buffer::ByteBuffer; use vortex_buffer::ByteBufferMut; use vortex_error::VortexResult; +use crate::FSST; use crate::FSSTData; pub(super) fn canonicalize_fsst( - array: &FSSTData, + array: ArrayView<'_, FSST>, ctx: &mut ExecutionCtx, ) -> VortexResult { - let (buffers, views) = fsst_decode_views(array, 0, ctx)?; + let (buffers, views) = fsst_decode_views(array.data(), 0, ctx)?; // SAFETY: FSST already validates the bytes for binary/UTF-8. We build views directly on // top of them, so the view pointers will all be valid. Ok(unsafe { diff --git a/encodings/fsst/src/compress.rs b/encodings/fsst/src/compress.rs index eb016789584..53772e82636 100644 --- a/encodings/fsst/src/compress.rs +++ b/encodings/fsst/src/compress.rs @@ -14,8 +14,8 @@ use vortex_buffer::BufferMut; use vortex_error::VortexExpect; /// Compress a string array using FSST. +use crate::FSST; use crate::FSSTArray; -use crate::FSSTData; pub fn fsst_compress>( strings: A, len: usize, @@ -102,11 +102,8 @@ where let uncompressed_lengths = uncompressed_lengths.into_array(); - FSSTArray::try_from_data( - FSSTData::try_new(dtype, symbols, symbol_lengths, codes, uncompressed_lengths) - .vortex_expect("building FSSTArray from parts"), - ) - .vortex_expect("FSSTData is always valid") + FSST::try_new(dtype, symbols, symbol_lengths, codes, uncompressed_lengths) + .vortex_expect("FSST parts must be valid") } #[cfg(test)] diff --git a/encodings/fsst/src/compute/cast.rs b/encodings/fsst/src/compute/cast.rs index 496d5bf3f92..82034daab21 100644 --- a/encodings/fsst/src/compute/cast.rs +++ b/encodings/fsst/src/compute/cast.rs @@ -11,7 +11,6 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::FSST; -use crate::FSSTData; impl CastReduce for FSST { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // FSST is a string compression encoding. @@ -25,7 +24,7 @@ impl CastReduce for FSST { .cast(array.codes().dtype().with_nullability(dtype.nullability()))?; Ok(Some( - FSSTData::try_new( + FSST::try_new( dtype.clone(), array.symbols().clone(), array.symbol_lengths().clone(), diff --git a/encodings/fsst/src/compute/compare.rs b/encodings/fsst/src/compute/compare.rs index cc0c563a887..14ea2cbe4d6 100644 --- a/encodings/fsst/src/compute/compare.rs +++ b/encodings/fsst/src/compute/compare.rs @@ -21,8 +21,6 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use crate::FSST; -use crate::FSSTData; - impl CompareKernel for FSST { fn compare( lhs: ArrayView<'_, Self>, @@ -31,7 +29,7 @@ impl CompareKernel for FSST { ctx: &mut ExecutionCtx, ) -> VortexResult> { match rhs.as_constant() { - Some(constant) => compare_fsst_constant(&lhs, &constant, operator, ctx), + Some(constant) => compare_fsst_constant(lhs, &constant, operator, ctx), // Otherwise, fall back to the default comparison behavior. _ => Ok(None), } @@ -40,7 +38,7 @@ impl CompareKernel for FSST { /// Specialized compare function implementation used when performing against a constant fn compare_fsst_constant( - left: &FSSTData, + left: ArrayView<'_, FSST>, right: &Scalar, operator: CompareOperator, ctx: &mut ExecutionCtx, @@ -78,7 +76,7 @@ fn compare_fsst_constant( return Ok(Some( BoolArray::new( buffer, - Validity::copy_from_array(&left.clone().into_array())? + Validity::copy_from_array(left.array())? .union_nullability(right.dtype().nullability()), ) .into_array(), diff --git a/encodings/fsst/src/compute/filter.rs b/encodings/fsst/src/compute/filter.rs index de58a83d5c4..0a2aa130d7a 100644 --- a/encodings/fsst/src/compute/filter.rs +++ b/encodings/fsst/src/compute/filter.rs @@ -12,7 +12,6 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::FSST; -use crate::FSSTData; impl FilterKernel for FSST { fn filter( @@ -31,7 +30,7 @@ impl FilterKernel for FSST { .vortex_expect("must be VarBin"); Ok(Some( - FSSTData::try_new( + FSST::try_new( array.dtype().clone(), array.symbols().clone(), array.symbol_lengths().clone(), diff --git a/encodings/fsst/src/compute/mod.rs b/encodings/fsst/src/compute/mod.rs index 7ca91c4c331..1bc1df71a29 100644 --- a/encodings/fsst/src/compute/mod.rs +++ b/encodings/fsst/src/compute/mod.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use crate::FSSTData; mod cast; mod compare; mod filter; @@ -28,7 +27,7 @@ impl TakeExecute for FSST { _ctx: &mut ExecutionCtx, ) -> VortexResult> { Ok(Some( - FSSTData::try_new( + FSST::try_new( array .dtype() .clone() diff --git a/encodings/fsst/src/slice.rs b/encodings/fsst/src/slice.rs index b30e574e9bd..5e0dd21a9f3 100644 --- a/encodings/fsst/src/slice.rs +++ b/encodings/fsst/src/slice.rs @@ -12,14 +12,13 @@ use vortex_error::VortexResult; use vortex_error::vortex_err; use crate::FSST; -use crate::FSSTData; impl SliceReduce for FSST { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { // SAFETY: slicing the `codes` leaves the symbol table intact Ok(Some( unsafe { - FSSTData::new_unchecked( + FSST::new_unchecked( array.dtype().clone(), array.symbols().clone(), array.symbol_lengths().clone(), diff --git a/encodings/parquet-variant/src/array.rs b/encodings/parquet-variant/src/array.rs index f07ae0e847f..df872889391 100644 --- a/encodings/parquet-variant/src/array.rs +++ b/encodings/parquet-variant/src/array.rs @@ -7,6 +7,8 @@ use arrow_array::Array as ArrowArray; use arrow_array::ArrayRef as ArrowArrayRef; use arrow_schema::Field; use parquet_variant_compute::VariantArray as ArrowVariantArray; +use vortex_array::Array; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; @@ -15,7 +17,6 @@ use vortex_array::arrow::ArrowArrayExecutor; use vortex_array::arrow::FromArrowArray; use vortex_array::arrow::to_arrow_null_buffer; use vortex_array::dtype::DType; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable::validity_to_child; use vortex_buffer::BitBuffer; @@ -24,6 +25,8 @@ use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_ensure_eq; +use crate::ParquetVariant; + /// The validity bitmap indicating which elements are non-null. pub(crate) const VALIDITY_SLOT: usize = 0; /// The metadata array for the Parquet variant values. @@ -63,10 +66,24 @@ pub(crate) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity", "metadata", "value /// (which includes nullability). #[derive(Clone, Debug)] pub struct ParquetVariantData { - pub(crate) dtype: DType, pub(crate) validity: Validity, pub(crate) slots: Vec>, - pub(crate) stats_set: ArrayStats, +} + +impl ParquetVariant { + pub fn try_new( + validity: Validity, + metadata: ArrayRef, + value: Option, + typed_value: Option, + ) -> VortexResult> { + let len = metadata.len(); + let dtype = DType::Variant(validity.nullability()); + let data = ParquetVariantData::try_new(validity, metadata, value, typed_value)?; + Ok(unsafe { + Array::from_parts_unchecked(ArrayParts::new(ParquetVariant, dtype, len, data)) + }) + } } impl ParquetVariantData { @@ -77,39 +94,82 @@ impl ParquetVariantData { value: Option, typed_value: Option, ) -> VortexResult { + let len = metadata.len(); + let dtype = DType::Variant(validity.nullability()); + Self::validate_parts( + &validity, + &metadata, + value.as_ref(), + typed_value.as_ref(), + &dtype, + len, + )?; + + let validity_child = validity_to_child(&validity, len); + let slots = vec![validity_child, Some(metadata), value, typed_value]; + + Ok(Self { validity, slots }) + } + + pub(crate) fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { + Self::validate_parts( + &self.validity, + self.metadata_array(), + self.value_array(), + self.typed_value_array(), + dtype, + len, + ) + } + + fn validate_parts( + validity: &Validity, + metadata: &ArrayRef, + value: Option<&ArrayRef>, + typed_value: Option<&ArrayRef>, + dtype: &DType, + len: usize, + ) -> VortexResult<()> { + vortex_ensure!( + matches!(dtype, DType::Variant(_)), + "Expected Variant DType, found {dtype}" + ); vortex_ensure!( value.is_some() || typed_value.is_some(), "at least one of value or typed_value must be present" ); - let len = metadata.len(); + + vortex_ensure_eq!( + dtype.nullability(), + validity.nullability(), + "variant dtype nullability must match validity nullability" + ); + vortex_ensure_eq!( + metadata.dtype(), + &DType::Binary(vortex_array::dtype::Nullability::NonNullable), + "metadata dtype must be non-nullable binary" + ); + vortex_ensure_eq!( + metadata.len(), + len, + "metadata length must match array length" + ); + if let Some(validity_len) = validity.maybe_len() { - vortex_ensure_eq!( - validity_len, - len, - "validity length must match metadata length" - ); - } - if let Some(ref v) = value { - vortex_ensure_eq!(v.len(), len, "value length must match metadata length"); + vortex_ensure_eq!(validity_len, len, "validity length must match array length"); } - if let Some(ref tv) = typed_value { - vortex_ensure_eq!( - tv.len(), - len, - "typed_value length must match metadata length" + if let Some(v) = value { + vortex_ensure!( + matches!(v.dtype(), DType::Binary(_)), + "value dtype must be binary, found {}", + v.dtype() ); + vortex_ensure_eq!(v.len(), len, "value length must match array length"); } - let nullability = validity.nullability(); - - let validity_child = validity_to_child(&validity, len); - let slots = vec![validity_child, Some(metadata), value, typed_value]; - - Ok(Self { - dtype: DType::Variant(nullability), - validity, - slots, - stats_set: ArrayStats::default(), - }) + if let Some(tv) = typed_value { + vortex_ensure_eq!(tv.len(), len, "typed_value length must match array length"); + } + Ok(()) } /// Returns a reference to the metadata child array. @@ -168,7 +228,7 @@ impl ParquetVariantData { .map(|tv| ArrayRef::from_arrow(tv.as_ref(), typed_value_nullable)) .transpose()?; - let pv = ParquetVariantData::try_new(validity, metadata, value, typed_value)?; + let pv = ParquetVariant::try_new(validity, metadata, value, typed_value)?; Ok(VariantArray::new(pv.into_array()).into_array()) } diff --git a/encodings/parquet-variant/src/kernel.rs b/encodings/parquet-variant/src/kernel.rs index d921f97a325..69e922e1548 100644 --- a/encodings/parquet-variant/src/kernel.rs +++ b/encodings/parquet-variant/src/kernel.rs @@ -18,7 +18,6 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::ParquetVariant; -use crate::array::ParquetVariantData; pub(crate) static PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ ParentKernelSet::lift(&FilterExecuteAdaptor(ParquetVariant)), @@ -43,7 +42,7 @@ impl SliceKernel for ParquetVariant { .map(|tv| tv.slice(range)) .transpose()?; Ok(Some( - ParquetVariantData::try_new(validity, metadata, value, typed_value)?.into_array(), + ParquetVariant::try_new(validity, metadata, value, typed_value)?.into_array(), )) } } @@ -65,7 +64,7 @@ impl FilterKernel for ParquetVariant { .map(|tv| tv.filter(mask.clone())) .transpose()?; Ok(Some( - ParquetVariantData::try_new(validity, metadata, value, typed_value)?.into_array(), + ParquetVariant::try_new(validity, metadata, value, typed_value)?.into_array(), )) } } @@ -87,7 +86,7 @@ impl TakeExecute for ParquetVariant { .map(|tv| tv.take(indices.clone())) .transpose()?; Ok(Some( - ParquetVariantData::try_new(validity, metadata, value, typed_value)?.into_array(), + ParquetVariant::try_new(validity, metadata, value, typed_value)?.into_array(), )) } } diff --git a/encodings/parquet-variant/src/lib.rs b/encodings/parquet-variant/src/lib.rs index 3f1f7cb68a6..42c97d28870 100644 --- a/encodings/parquet-variant/src/lib.rs +++ b/encodings/parquet-variant/src/lib.rs @@ -33,4 +33,3 @@ mod vtable; pub use array::ParquetVariantData; pub use vtable::ParquetVariant; pub use vtable::ParquetVariantArray; -pub use vtable::ParquetVariantMetadata; diff --git a/encodings/parquet-variant/src/operations.rs b/encodings/parquet-variant/src/operations.rs index 6ccbfeb6d75..4b2d26b77a4 100644 --- a/encodings/parquet-variant/src/operations.rs +++ b/encodings/parquet-variant/src/operations.rs @@ -61,7 +61,7 @@ impl OperationsVTable for ParquetVariant { )?; Scalar::try_new( - array.dtype.clone(), + array.dtype().clone(), Some(ScalarValue::Variant(Box::new(inner))), ) } diff --git a/encodings/parquet-variant/src/vtable.rs b/encodings/parquet-variant/src/vtable.rs index 8815734e0c9..083cfc42261 100644 --- a/encodings/parquet-variant/src/vtable.rs +++ b/encodings/parquet-variant/src/vtable.rs @@ -20,7 +20,6 @@ use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; @@ -46,23 +45,6 @@ impl ParquetVariant { pub const ID: ArrayId = ArrayId::new_ref("vortex.parquet.variant"); } -/// Serialized metadata for a [`ParquetVariantArray`]. -#[derive(Clone, Debug)] -pub struct ParquetVariantMetadata { - /// Whether the un-shredded `value` child is present. - pub has_value: bool, - /// Whether the `value` child is nullable. - /// - /// In partially-shredded layouts, rows whose data lives entirely in `typed_value` have a - /// null `value` slot, so the Arrow field is marked nullable. This flag preserves that - /// distinction across serialization round-trips. - pub value_nullable: bool, - /// DType of the shredded `typed_value`, if present. - /// - /// This is required to deserialize non-variant shredded children. - pub typed_value_dtype: Option, -} - #[derive(Clone, prost::Message)] struct ParquetVariantMetadataProto { /// Whether the un-shredded `value` child is present. @@ -80,28 +62,15 @@ vtable!(ParquetVariant, ParquetVariant, ParquetVariantData); impl VTable for ParquetVariant { type ArrayData = ParquetVariantData; - type Metadata = ParquetVariantMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromValidityHelper; - fn vtable(_array: &Self::ArrayData) -> &Self { - &ParquetVariant - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ParquetVariantData) -> usize { - array.metadata_array().len() - } - - fn dtype(array: &ParquetVariantData) -> &DType { - &array.dtype - } - - fn stats(array: &ParquetVariantData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate(dtype, len) } fn array_hash(array: &ParquetVariantData, state: &mut H, precision: Precision) { @@ -167,64 +136,50 @@ impl VTable for ParquetVariant { SLOT_NAMES[idx].to_string() } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ParquetVariantMetadata { - has_value: array.value_array().is_some(), - value_nullable: array.value_array().is_some_and(|v| v.dtype().is_nullable()), - typed_value_dtype: array.typed_value_array().map(|tv| tv.dtype().clone()), - }) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - let typed_value_dtype = metadata - .typed_value_dtype - .as_ref() - .map(|dtype| dtype.try_into()) + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + let typed_value_dtype = array + .typed_value_array() + .map(|tv| tv.dtype().try_into()) .transpose()?; Ok(Some( ParquetVariantMetadataProto { - has_value: metadata.has_value, + has_value: array.value_array().is_some(), typed_value_dtype, - value_nullable: metadata.value_nullable, + value_nullable: array.value_array().is_some_and(|v| v.dtype().is_nullable()), } .encode_to_vec(), )) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], + &self, + dtype: &DType, + len: usize, + metadata: &[u8], + buffers: &[BufferHandle], + children: &dyn ArrayChildren, session: &VortexSession, - ) -> VortexResult { - let proto = ParquetVariantMetadataProto::decode(bytes)?; + ) -> VortexResult { + vortex_ensure!( + buffers.is_empty(), + "ParquetVariantArray expects 0 buffers, got {}", + buffers.len() + ); + + let proto = ParquetVariantMetadataProto::decode(metadata)?; let typed_value_dtype = match proto.typed_value_dtype.as_ref() { Some(dtype) => Some(DType::from_proto(dtype, session)?), None => None, }; - Ok(ParquetVariantMetadata { - has_value: proto.has_value, - value_nullable: proto.value_nullable, - typed_value_dtype, - }) - } - fn build( - dtype: &DType, - len: usize, - metadata: &Self::Metadata, - _buffers: &[BufferHandle], - children: &dyn ArrayChildren, - ) -> VortexResult { vortex_ensure!(matches!(dtype, DType::Variant(_)), "Expected Variant DType"); - let has_typed_value = metadata.typed_value_dtype.is_some(); + let has_typed_value = typed_value_dtype.is_some(); vortex_ensure!( - metadata.has_value || has_typed_value, + proto.has_value || has_typed_value, "At least one of value or typed_value must be present" ); - let expected_children = 1 + metadata.has_value as usize + has_typed_value as usize; + let expected_children = 1 + proto.has_value as usize + has_typed_value as usize; vortex_ensure!( children.len() == expected_children || children.len() == expected_children + 1, "Expected {} or {} children, got {}", @@ -242,12 +197,8 @@ impl VTable for ParquetVariant { children.get(child_idx, &DType::Binary(Nullability::NonNullable), len)?; child_idx += 1; - let value = if metadata.has_value { - let v = children.get( - child_idx, - &DType::Binary(metadata.value_nullable.into()), - len, - )?; + let value = if proto.has_value { + let v = children.get(child_idx, &DType::Binary(proto.value_nullable.into()), len)?; child_idx += 1; Some(v) } else { @@ -256,9 +207,7 @@ impl VTable for ParquetVariant { let typed_value = if has_typed_value { // typed_value can be any type — primitive, list, struct, etc. - let dtype = metadata - .typed_value_dtype - .clone() + let dtype = typed_value_dtype .ok_or_else(|| vortex_err!("typed_value_dtype missing for typed_value child"))?; let tv = children.get(child_idx, &dtype, len)?; Some(tv) @@ -313,8 +262,8 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::serde::ArrayParts; use vortex_array::serde::SerializeOptions; + use vortex_array::serde::SerializedArray; use vortex_array::session::ArraySessionExt; use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; @@ -324,8 +273,6 @@ mod tests { use vortex_session::registry::ReadContext; use crate::ParquetVariant; - use crate::ParquetVariantData; - fn roundtrip(array: ArrayRef) -> ArrayRef { let dtype = array.dtype().clone(); let len = array.len(); @@ -343,7 +290,7 @@ mod tests { session.arrays().register(ParquetVariant); session.arrays().register(Variant); - let parts = ArrayParts::try_from(concat).unwrap(); + let parts = SerializedArray::try_from(concat).unwrap(); parts .decode(&dtype, len, &ReadContext::new(ctx.to_ids()), &session) .unwrap() @@ -357,7 +304,7 @@ mod tests { let inner_metadata = VarBinViewArray::from_iter_bin([b"\x01\x00", b"\x01\x00", b"\x01\x00"]).into_array(); let inner_value = VarBinViewArray::from_iter_bin([b"\x02", b"\x03", b"\x04"]).into_array(); - let inner_pv = ParquetVariantData::try_new( + let inner_pv = ParquetVariant::try_new( Validity::NonNullable, inner_metadata, Some(inner_value), @@ -366,7 +313,7 @@ mod tests { .unwrap(); let typed_value = VariantArray::new(inner_pv.into_array()).into_array(); - let outer_pv = ParquetVariantData::try_new( + let outer_pv = ParquetVariant::try_new( Validity::NonNullable, outer_metadata, None, @@ -389,7 +336,7 @@ mod tests { let value = VarBinViewArray::from_iter_bin([b"\x10", b"\x11", b"\x12"]).into_array(); let validity = Validity::from(BitBuffer::from_iter([true, false, true])); - let pv = ParquetVariantData::try_new(validity, metadata, Some(value), None).unwrap(); + let pv = ParquetVariant::try_new(validity, metadata, Some(value), None).unwrap(); let array = pv.into_array(); let decoded = roundtrip(array.clone()); @@ -406,7 +353,7 @@ mod tests { VarBinViewArray::from_iter_bin([b"\x01\x00", b"\x01\x00", b"\x01\x00"]).into_array(); let typed_value = buffer![10i32, 20, 30].into_array(); - let outer_pv = ParquetVariantData::try_new( + let outer_pv = ParquetVariant::try_new( Validity::NonNullable, outer_metadata, None, diff --git a/encodings/pco/public-api.lock b/encodings/pco/public-api.lock index 21d9c41cec2..4513321e259 100644 --- a/encodings/pco/public-api.lock +++ b/encodings/pco/public-api.lock @@ -20,8 +20,6 @@ impl vortex_array::array::vtable::VTable for vortex_pco::Pco pub type vortex_pco::Pco::ArrayData = vortex_pco::PcoData -pub type vortex_pco::Pco::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_pco::Pco::OperationsVTable = vortex_pco::Pco pub type vortex_pco::Pco::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromValiditySliceHelper @@ -34,33 +32,23 @@ pub fn vortex_pco::Pco::buffer(array: vortex_array::array::view::ArrayView<'_, S pub fn vortex_pco::Pco::buffer_name(array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_pco::Pco::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_pco::Pco::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_pco::Pco::dtype(array: &vortex_pco::PcoData) -> &vortex_array::dtype::DType +pub fn vortex_pco::Pco::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_pco::Pco::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_pco::Pco::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_pco::Pco::len(array: &vortex_pco::PcoData) -> usize - -pub fn vortex_pco::Pco::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_pco::Pco::nbuffers(array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_pco::Pco::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_pco::Pco::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_pco::Pco::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_pco::Pco::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_pco::Pco::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_pco::Pco::stats(array: &vortex_pco::PcoData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_pco::Pco::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_pco::Pco::validate(&self, data: &vortex_pco::PcoData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_pco::Pco::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -104,8 +92,6 @@ impl vortex_pco::PcoData pub fn vortex_pco::PcoData::decompress(&self, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_pco::PcoData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_pco::PcoData::from_array(array: vortex_array::array::erased::ArrayRef, level: usize, nums_per_page: usize) -> vortex_error::VortexResult pub fn vortex_pco::PcoData::from_primitive(parray: &vortex_array::arrays::primitive::vtable::PrimitiveArray, level: usize, values_per_page: usize) -> vortex_error::VortexResult @@ -114,24 +100,18 @@ pub fn vortex_pco::PcoData::is_empty(&self) -> bool pub fn vortex_pco::PcoData::len(&self) -> usize -pub fn vortex_pco::PcoData::new(chunk_metas: alloc::vec::Vec, pages: alloc::vec::Vec, dtype: vortex_array::dtype::DType, metadata: vortex_pco::PcoMetadata, len: usize, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_pco::PcoData::new(chunk_metas: alloc::vec::Vec, pages: alloc::vec::Vec, ptype: vortex_array::dtype::ptype::PType, metadata: vortex_pco::PcoMetadata, len: usize, validity: vortex_array::validity::Validity) -> Self + +pub fn vortex_pco::PcoData::validate(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> impl core::clone::Clone for vortex_pco::PcoData pub fn vortex_pco::PcoData::clone(&self) -> vortex_pco::PcoData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_pco::PcoData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_pco::PcoData pub fn vortex_pco::PcoData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_pco::PcoData - -pub fn vortex_pco::PcoData::into_array(self) -> vortex_array::array::erased::ArrayRef - impl vortex_array::array::vtable::validity::ValiditySliceHelper for vortex_pco::PcoData pub fn vortex_pco::PcoData::unsliced_validity_and_slice(&self) -> (&vortex_array::validity::Validity, usize, usize) diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index 1365d8ad087..265566ec35e 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -19,6 +19,7 @@ use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; @@ -26,7 +27,6 @@ use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; use vortex_array::Precision; -use vortex_array::ProstMetadata; use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::Primitive; @@ -37,7 +37,6 @@ use vortex_array::dtype::PType; use vortex_array::dtype::half; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; @@ -84,28 +83,15 @@ vtable!(Pco, Pco, PcoData); impl VTable for Pco { type ArrayData = PcoData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromValiditySliceHelper; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Pco - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &PcoData) -> usize { - array.slice_stop - array.slice_start - } - - fn dtype(array: &PcoData) -> &DType { - &array.dtype - } - - fn stats(array: &PcoData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &PcoData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate(dtype, len) } fn array_hash(array: &PcoData, state: &mut H, precision: Precision) { @@ -168,31 +154,20 @@ impl VTable for Pco { } } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(array.metadata.clone())) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.0.encode_to_vec())) + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some(array.metadata.clone().encode_to_vec())) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata(PcoMetadata::decode(bytes)?)) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = PcoMetadata::decode(metadata)?; let validity = if children.is_empty() { Validity::from(dtype.nullability()) } else if children.len() == 1 { @@ -202,18 +177,17 @@ impl VTable for Pco { vortex_bail!("PcoArray expected 0 or 1 child, got {}", children.len()); }; - vortex_ensure!(buffers.len() >= metadata.0.chunks.len()); - let chunk_metas = buffers[..metadata.0.chunks.len()] + vortex_ensure!(buffers.len() >= metadata.chunks.len()); + let chunk_metas = buffers[..metadata.chunks.len()] .iter() .map(|b| b.clone().try_to_host_sync()) .collect::>>()?; - let pages = buffers[metadata.0.chunks.len()..] + let pages = buffers[metadata.chunks.len()..] .iter() .map(|b| b.clone().try_to_host_sync()) .collect::>>()?; let expected_n_pages = metadata - .0 .chunks .iter() .map(|info| info.pages.len()) @@ -223,8 +197,8 @@ impl VTable for Pco { Ok(PcoData::new( chunk_metas, pages, - dtype.clone(), - metadata.0.clone(), + dtype.as_ptype(), + metadata, len, validity, )) @@ -247,7 +221,7 @@ impl VTable for Pco { ); array.unsliced_validity = match &slots[VALIDITY_SLOT] { Some(arr) => Validity::Array(arr.clone()), - None => Validity::from(array.dtype.nullability()), + None => Validity::from(array.unsliced_validity.nullability()), }; array.slots = slots; Ok(()) @@ -267,7 +241,10 @@ impl VTable for Pco { } pub(crate) fn number_type_from_dtype(dtype: &DType) -> NumberType { - let ptype = dtype.as_ptype(); + number_type_from_ptype(dtype.as_ptype()) +} + +pub(crate) fn number_type_from_ptype(ptype: PType) -> NumberType { match ptype { PType::F16 => NumberType::F16, PType::F32 => NumberType::F32, @@ -302,13 +279,21 @@ pub struct Pco; impl Pco { pub const ID: ArrayId = ArrayId::new_ref("vortex.pco"); + pub(crate) fn try_new(dtype: DType, data: PcoData) -> VortexResult { + let len = data.len(); + data.validate(&dtype, len)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Pco, dtype, len, data)) }) + } + /// Compress a primitive array using pcodec. pub fn from_primitive( parray: &PrimitiveArray, level: usize, values_per_page: usize, ) -> VortexResult { - Array::try_from_data(PcoData::from_primitive(parray, level, values_per_page)?) + let dtype = parray.dtype().clone(); + let data = PcoData::from_primitive(parray, level, values_per_page)?; + Self::try_new(dtype, data) } } @@ -322,20 +307,72 @@ pub struct PcoData { pub(crate) chunk_metas: Vec, pub(crate) pages: Vec, pub(crate) metadata: PcoMetadata, - dtype: DType, + ptype: PType, pub(crate) unsliced_validity: Validity, unsliced_n_rows: usize, pub(super) slots: Vec>, - stats_set: ArrayStats, slice_start: usize, slice_stop: usize, } impl PcoData { + pub fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { + let _ = number_type_from_ptype(self.ptype); + vortex_ensure!( + dtype.as_ptype() == self.ptype, + "expected ptype {}, got {}", + self.ptype, + dtype.as_ptype() + ); + vortex_ensure!( + dtype.nullability() == self.unsliced_validity.nullability(), + "expected nullability {}, got {}", + self.unsliced_validity.nullability(), + dtype.nullability() + ); + vortex_ensure!( + self.slice_start <= self.slice_stop && self.slice_stop <= self.unsliced_n_rows, + "invalid slice range {}..{} for {} rows", + self.slice_start, + self.slice_stop, + self.unsliced_n_rows + ); + vortex_ensure!( + self.slice_stop - self.slice_start == len, + "expected len {len}, got {}", + self.slice_stop - self.slice_start + ); + if let Some(validity_len) = self.unsliced_validity.maybe_len() { + vortex_ensure!( + validity_len == self.unsliced_n_rows, + "expected validity len {}, got {}", + self.unsliced_n_rows, + validity_len + ); + } + vortex_ensure!( + self.chunk_metas.len() == self.metadata.chunks.len(), + "expected {} chunk metas, got {}", + self.metadata.chunks.len(), + self.chunk_metas.len() + ); + vortex_ensure!( + self.pages.len() + == self + .metadata + .chunks + .iter() + .map(|chunk| chunk.pages.len()) + .sum::(), + "page count does not match metadata" + ); + Ok(()) + } + pub fn new( chunk_metas: Vec, pages: Vec, - dtype: DType, + ptype: PType, metadata: PcoMetadata, len: usize, validity: Validity, @@ -346,11 +383,10 @@ impl PcoData { chunk_metas, pages, metadata, - dtype, + ptype, unsliced_validity: validity, unsliced_n_rows: len, slots: vec![validity_slot], - stats_set: Default::default(), slice_start: 0, slice_stop: len, } @@ -430,7 +466,7 @@ impl PcoData { Ok(PcoData::new( chunk_meta_buffers, page_buffers, - parray.dtype().clone(), + parray.dtype().as_ptype(), metadata, parray.len(), parray.validity(), @@ -450,7 +486,7 @@ impl PcoData { pub fn decompress(&self, ctx: &mut ExecutionCtx) -> VortexResult { // To start, we figure out which chunks and pages we need to decompress, and with // what value offset into the first such page. - let number_type = number_type_from_dtype(&self.dtype); + let number_type = number_type_from_ptype(self.ptype); let values_byte_buffer = match_number_enum!( number_type, NumberType => { @@ -460,7 +496,7 @@ impl PcoData { Ok(PrimitiveArray::from_values_byte_buffer( values_byte_buffer, - self.dtype.as_ptype(), + self.ptype, self.unsliced_validity .slice(self.slice_start..self.slice_stop)?, self.slice_stop - self.slice_start, @@ -547,7 +583,6 @@ impl PcoData { PcoData { slice_start: self.slice_start + start, slice_stop: self.slice_start + stop, - stats_set: Default::default(), ..self.clone() } } @@ -562,11 +597,6 @@ impl PcoData { self.slice_stop == self.slice_start } - /// Returns the logical data type of the array. - pub fn dtype(&self) -> &DType { - &self.dtype - } - pub(crate) fn slice_start(&self) -> usize { self.slice_start } diff --git a/encodings/pco/src/compute/cast.rs b/encodings/pco/src/compute/cast.rs index 378f07254ac..877535bbd97 100644 --- a/encodings/pco/src/compute/cast.rs +++ b/encodings/pco/src/compute/cast.rs @@ -29,18 +29,17 @@ impl CastReduce for Pco { .clone() .cast_nullability(dtype.nullability(), array.len())?; - return Ok(Some( - PcoData::new( - array.chunk_metas.clone(), - array.pages.clone(), - dtype.clone(), - array.metadata.clone(), - array.unsliced_n_rows(), - new_validity, - ) - ._slice(array.slice_start(), array.slice_stop()) - .into_array(), - )); + let data = PcoData::new( + array.chunk_metas.clone(), + array.pages.clone(), + dtype.as_ptype(), + array.metadata.clone(), + array.unsliced_n_rows(), + new_validity, + ) + ._slice(array.slice_start(), array.slice_stop()); + + return Ok(Some(Pco::try_new(dtype.clone(), data)?.into_array())); } // For other casts (e.g., numeric type changes), decode to canonical and let PrimitiveArray handle it diff --git a/encodings/pco/src/slice.rs b/encodings/pco/src/slice.rs index bdaa1006294..a53a1e39898 100644 --- a/encodings/pco/src/slice.rs +++ b/encodings/pco/src/slice.rs @@ -13,6 +13,8 @@ use crate::Pco; impl SliceReduce for Pco { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - Ok(Some(array._slice(range.start, range.end).into_array())) + Ok(Some( + Pco::try_new(array.dtype().clone(), array._slice(range.start, range.end))?.into_array(), + )) } } diff --git a/encodings/pco/src/test.rs b/encodings/pco/src/test.rs index bfadb843fd4..0bcbd8837ee 100644 --- a/encodings/pco/src/test.rs +++ b/encodings/pco/src/test.rs @@ -17,8 +17,8 @@ use vortex_array::assert_nth_scalar; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; -use vortex_array::serde::ArrayParts; use vortex_array::serde::SerializeOptions; +use vortex_array::serde::SerializedArray; use vortex_array::session::ArraySession; use vortex_array::session::ArraySessionExt; use vortex_array::validity::Validity; @@ -39,8 +39,6 @@ static SESSION: LazyLock = LazyLock::new(|| { }); use crate::Pco; -use crate::PcoArray; - #[test] fn test_compress_decompress() { let data: Vec = (0..200).collect(); @@ -101,7 +99,8 @@ fn test_validity_and_multiple_chunks_and_pages() { let compression_level = 3; let values_per_chunk = 33; let values_per_page = 10; - let compressed = PcoArray::try_from_data( + let compressed = Pco::try_new( + array.dtype().clone(), PcoData::from_primitive_with_values_per_chunk( &array, compression_level, @@ -179,7 +178,7 @@ fn test_serde() -> VortexResult<()> { .collect::>() .freeze(); - let parts = ArrayParts::try_from(bytes)?; + let parts = SerializedArray::try_from(bytes)?; let decoded = parts.decode( &DType::Primitive(PType::I32, Nullability::NonNullable), 1_000_000, diff --git a/encodings/runend/public-api.lock b/encodings/runend/public-api.lock index a3736617650..528e3916681 100644 --- a/encodings/runend/public-api.lock +++ b/encodings/runend/public-api.lock @@ -44,8 +44,6 @@ impl vortex_array::array::vtable::VTable for vortex_runend::RunEnd pub type vortex_runend::RunEnd::ArrayData = vortex_runend::RunEndData -pub type vortex_runend::RunEnd::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_runend::RunEnd::OperationsVTable = vortex_runend::RunEnd pub type vortex_runend::RunEnd::ValidityVTable = vortex_runend::RunEnd @@ -58,11 +56,7 @@ pub fn vortex_runend::RunEnd::buffer(_array: vortex_array::array::view::ArrayVie pub fn vortex_runend::RunEnd::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_runend::RunEnd::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_runend::RunEnd::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_runend::RunEnd::dtype(array: &vortex_runend::RunEndData) -> &vortex_array::dtype::DType +pub fn vortex_runend::RunEnd::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_runend::RunEnd::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -70,23 +64,17 @@ pub fn vortex_runend::RunEnd::execute_parent(array: vortex_array::array::view::A pub fn vortex_runend::RunEnd::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_runend::RunEnd::len(array: &vortex_runend::RunEndData) -> usize - -pub fn vortex_runend::RunEnd::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_runend::RunEnd::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_runend::RunEnd::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_runend::RunEnd::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_runend::RunEnd::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_runend::RunEnd::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_runend::RunEnd::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_runend::RunEnd::stats(array: &vortex_runend::RunEndData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_runend::RunEnd::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_runend::RunEnd::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_runend::RunEnd::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -118,12 +106,6 @@ impl vortex_array::scalar_fn::fns::fill_null::kernel::FillNullReduce for vortex_ pub fn vortex_runend::RunEnd::fill_null(array: vortex_array::array::view::ArrayView<'_, Self>, fill_value: &vortex_array::scalar::Scalar) -> vortex_error::VortexResult> -pub struct vortex_runend::RunEndArrayParts - -pub vortex_runend::RunEndArrayParts::ends: vortex_array::array::erased::ArrayRef - -pub vortex_runend::RunEndArrayParts::values: vortex_array::array::erased::ArrayRef - pub struct vortex_runend::RunEndData impl vortex_runend::RunEndData @@ -136,44 +118,36 @@ pub fn vortex_runend::RunEndData::ends(&self) -> &vortex_array::array::erased::A pub fn vortex_runend::RunEndData::find_physical_index(&self, index: usize) -> vortex_error::VortexResult -pub fn vortex_runend::RunEndData::into_parts(self) -> vortex_runend::RunEndArrayParts - -pub fn vortex_runend::RunEndData::is_empty(&self) -> bool - -pub fn vortex_runend::RunEndData::len(&self) -> usize +pub fn vortex_runend::RunEndData::into_parts(self) -> vortex_runend::RunEndDataParts pub fn vortex_runend::RunEndData::new(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef) -> Self -pub unsafe fn vortex_runend::RunEndData::new_unchecked(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> Self +pub unsafe fn vortex_runend::RunEndData::new_unchecked(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, _length: usize) -> Self pub fn vortex_runend::RunEndData::offset(&self) -> usize -pub fn vortex_runend::RunEndData::try_new(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult - -pub fn vortex_runend::RunEndData::try_new_offset_length(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> vortex_error::VortexResult - pub fn vortex_runend::RunEndData::values(&self) -> &vortex_array::array::erased::ArrayRef impl core::clone::Clone for vortex_runend::RunEndData pub fn vortex_runend::RunEndData::clone(&self) -> vortex_runend::RunEndData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_runend::RunEndData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_runend::RunEndData pub fn vortex_runend::RunEndData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_runend::RunEndData - -pub fn vortex_runend::RunEndData::into_array(self) -> vortex_array::array::erased::ArrayRef - impl vortex_array::arrow::FromArrowArray<&arrow_array::array::run_array::RunArray> for vortex_runend::RunEndData where ::Native: vortex_array::dtype::ptype::NativePType pub fn vortex_runend::RunEndData::from_arrow(array: &arrow_array::array::run_array::RunArray, nullable: bool) -> vortex_error::VortexResult +pub struct vortex_runend::RunEndDataParts + +pub vortex_runend::RunEndDataParts::ends: vortex_array::array::erased::ArrayRef + +pub vortex_runend::RunEndDataParts::offset: usize + +pub vortex_runend::RunEndDataParts::values: vortex_array::array::erased::ArrayRef + pub struct vortex_runend::RunEndMetadata pub vortex_runend::RunEndMetadata::ends_ptype: i32 diff --git a/encodings/runend/src/arbitrary.rs b/encodings/runend/src/arbitrary.rs index a63fa9bb2ed..5347935c578 100644 --- a/encodings/runend/src/arbitrary.rs +++ b/encodings/runend/src/arbitrary.rs @@ -14,11 +14,12 @@ use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexExpect; -use crate::RunEndData; +use crate::RunEnd; +use crate::RunEndArray; /// A wrapper type to implement `Arbitrary` for `RunEndArray`. #[derive(Clone, Debug)] -pub struct ArbitraryRunEndArray(pub RunEndData); +pub struct ArbitraryRunEndArray(pub RunEndArray); impl<'a> Arbitrary<'a> for ArbitraryRunEndArray { fn arbitrary(u: &mut Unstructured<'a>) -> Result { @@ -42,7 +43,7 @@ impl ArbitraryRunEndArray { // Empty RunEndArray let ends = PrimitiveArray::from_iter(Vec::::new()).into_array(); let values = ArbitraryArray::arbitrary_with(u, Some(0), dtype)?.0; - let runend_array = RunEndData::try_new(ends, values) + let runend_array = RunEnd::try_new(ends, values) .vortex_expect("Empty RunEndArray creation should succeed"); return Ok(ArbitraryRunEndArray(runend_array)); } @@ -54,7 +55,7 @@ impl ArbitraryRunEndArray { // Each end must be > previous end, and first end must be >= 1 let ends = random_strictly_sorted_ends(u, num_runs, len)?; - let runend_array = RunEndData::try_new(ends, values) + let runend_array = RunEnd::try_new(ends, values) .vortex_expect("RunEndArray creation should succeed in arbitrary impl"); Ok(ArbitraryRunEndArray(runend_array)) diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index 081b48ab678..a9fe74e7d36 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -4,19 +4,18 @@ use std::fmt::Debug; use std::hash::Hash; +use prost::Message; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; -use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; -use vortex_array::SerializeMetadata; use vortex_array::arrays::Primitive; use vortex_array::arrays::VarBinViewArray; use vortex_array::buffer::BufferHandle; @@ -27,7 +26,6 @@ use vortex_array::scalar::PValue; use vortex_array::search_sorted::SearchSorted; use vortex_array::search_sorted::SearchSortedSide; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; @@ -61,28 +59,22 @@ pub struct RunEndMetadata { impl VTable for RunEnd { type ArrayData = RunEndData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &RunEnd - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &RunEndData) -> usize { - array.length - } - - fn dtype(array: &RunEndData) -> &DType { - array.values().dtype() - } - - fn stats(array: &RunEndData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + RunEndData::validate(data.ends(), data.values(), data.offset, len)?; + vortex_ensure!( + data.values().dtype() == dtype, + "expected dtype {}, got {}", + dtype, + data.values().dtype() + ); + Ok(()) } fn array_hash(array: &RunEndData, state: &mut H, precision: Precision) { @@ -109,37 +101,28 @@ impl VTable for RunEnd { vortex_panic!("RunEndArray buffer_name index {idx} out of bounds") } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(RunEndMetadata { - ends_ptype: PType::try_from(array.ends().dtype()).vortex_expect("Must be a valid PType") - as i32, - num_runs: array.ends().len() as u64, - offset: array.offset() as u64, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some( + RunEndMetadata { + ends_ptype: PType::try_from(array.ends().dtype()) + .vortex_expect("Must be a valid PType") as i32, + num_runs: array.ends().len() as u64, + offset: array.offset() as u64, + } + .encode_to_vec(), + )) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - let inner = as DeserializeMetadata>::deserialize(bytes)?; - Ok(ProstMetadata(inner)) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = RunEndMetadata::decode(metadata)?; let ends_dtype = DType::Primitive(metadata.ends_ptype(), Nullability::NonNullable); let runs = usize::try_from(metadata.num_runs).vortex_expect("Must be a valid usize"); let ends = children.get(0, &ends_dtype, runs)?; @@ -206,13 +189,12 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["ends", "values"]; pub struct RunEndData { pub(super) slots: Vec>, offset: usize, - length: usize, - stats_set: ArrayStats, } -pub struct RunEndArrayParts { +pub struct RunEndDataParts { pub ends: ArrayRef, pub values: ArrayRef, + pub offset: usize, } #[derive(Clone, Debug)] @@ -231,13 +213,17 @@ impl RunEnd { offset: usize, length: usize, ) -> RunEndArray { - Array::try_from_data(unsafe { RunEndData::new_unchecked(ends, values, offset, length) }) - .vortex_expect("RunEndData is always valid") + let data = unsafe { RunEndData::new_unchecked(ends, values, offset, length) }; + let dtype = data.dtype().clone(); + unsafe { Array::from_parts_unchecked(ArrayParts::new(RunEnd, dtype, length, data)) } } /// Build a new [`RunEndArray`] from ends and values. pub fn try_new(ends: ArrayRef, values: ArrayRef) -> VortexResult { - Array::try_from_data(RunEndData::try_new(ends, values)?) + let len = RunEndData::logical_len_from_ends(&ends)?; + let data = RunEndData::try_new_offset_length(ends, values, 0, len)?; + let dtype = data.dtype().clone(); + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(RunEnd, dtype, len, data)) }) } /// Build a new [`RunEndArray`] from ends, values, offset, and length. @@ -247,24 +233,34 @@ impl RunEnd { offset: usize, length: usize, ) -> VortexResult { - Array::try_from_data(RunEndData::try_new_offset_length( - ends, values, offset, length, - )?) + let data = RunEndData::try_new_offset_length(ends, values, offset, length)?; + let dtype = data.dtype().clone(); + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(RunEnd, dtype, length, data)) }) } /// Build a new [`RunEndArray`] from ends and values (panics on invalid input). pub fn new(ends: ArrayRef, values: ArrayRef) -> RunEndArray { - Array::try_from_data(RunEndData::new(ends, values)) - .vortex_expect("RunEndData is always valid") + Self::try_new(ends, values).vortex_expect("RunEndData is always valid") } /// Run the array through run-end encoding. pub fn encode(array: ArrayRef) -> VortexResult { - Array::try_from_data(RunEndData::encode(array)?) + let len = array.len(); + let data = RunEndData::encode(array)?; + let dtype = data.dtype().clone(); + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(RunEnd, dtype, len, data)) }) } } impl RunEndData { + fn logical_len_from_ends(ends: &ArrayRef) -> VortexResult { + if ends.is_empty() { + Ok(0) + } else { + usize::try_from(&ends.scalar_at(ends.len() - 1)?) + } + } + fn validate( ends: &ArrayRef, values: &ArrayRef, @@ -293,12 +289,8 @@ impl RunEndData { return Ok(()); } - // Avoid building a non-empty array with zero logical length. + // Zero-length logical slices may retain run metadata from the source array. if length == 0 { - vortex_ensure!( - ends.is_empty(), - "run ends must be empty when length is zero" - ); return Ok(()); } @@ -325,8 +317,8 @@ impl RunEndData { // Validate the offset and length are valid for the given ends and values if offset != 0 && length != 0 { let first_run_end = usize::try_from(&ends.scalar_at(0)?)?; - if first_run_end <= offset { - vortex_bail!("First run end {first_run_end} must be bigger than offset {offset}"); + if first_run_end < offset { + vortex_bail!("First run end {first_run_end} must be >= offset {offset}"); } } @@ -343,7 +335,7 @@ impl RunEndData { impl RunEndData { /// Build a new `RunEndArray` from an array of run `ends` and an array of `values`. /// - /// Panics if any of the validation conditions described in [`RunEndData::try_new`] is + /// Panics if any of the validation conditions described in [`RunEnd::try_new`] is /// not satisfied. /// /// # Examples @@ -375,20 +367,15 @@ impl RunEndData { /// # Validation /// /// The `ends` must be non-nullable unsigned integers. - pub fn try_new(ends: ArrayRef, values: ArrayRef) -> VortexResult { - let length: usize = if ends.is_empty() { - 0 - } else { - usize::try_from(&ends.scalar_at(ends.len() - 1)?)? - }; - + pub(crate) fn try_new(ends: ArrayRef, values: ArrayRef) -> VortexResult { + let length = Self::logical_len_from_ends(&ends)?; Self::try_new_offset_length(ends, values, 0, length) } /// Construct a new sliced `RunEndArray` with the provided offset and length. /// - /// This performs all the same validation as [`RunEndData::try_new`]. - pub fn try_new_offset_length( + /// This performs all the same validation as [`RunEnd::try_new_offset_length`]. + pub(crate) fn try_new_offset_length( ends: ArrayRef, values: ArrayRef, offset: usize, @@ -399,8 +386,6 @@ impl RunEndData { Ok(Self { slots: vec![Some(ends), Some(values)], offset, - length, - stats_set: Default::default(), }) } @@ -408,21 +393,20 @@ impl RunEndData { /// /// # Safety /// - /// The caller must ensure that all the validation performed in [`RunEndData::try_new`] is + /// The caller must ensure that all the validation performed in + /// [`RunEnd::try_new_offset_length`] is /// satisfied before calling this function. /// - /// See [`RunEndData::try_new`] for the preconditions needed to build a new array. + /// See [`RunEnd::try_new_offset_length`] for the preconditions needed to build a new array. pub unsafe fn new_unchecked( ends: ArrayRef, values: ArrayRef, offset: usize, - length: usize, + _length: usize, ) -> Self { Self { slots: vec![Some(ends), Some(values)], offset, - length, - stats_set: Default::default(), } } @@ -456,18 +440,6 @@ impl RunEndData { } } - /// Returns the length of the array. - #[inline] - pub fn len(&self) -> usize { - self.length - } - - /// Returns whether the array is empty. - #[inline] - pub fn is_empty(&self) -> bool { - self.length == 0 - } - /// Returns the logical data type of the array. #[inline] pub fn dtype(&self) -> &DType { @@ -504,16 +476,15 @@ impl RunEndData { .vortex_expect("RunEndArray values slot") } - /// Split an `RunEndArray` into parts. - #[inline] - pub fn into_parts(mut self) -> RunEndArrayParts { - RunEndArrayParts { + pub fn into_parts(mut self) -> RunEndDataParts { + RunEndDataParts { ends: self.slots[ENDS_SLOT] .take() .vortex_expect("RunEndArray ends slot"), values: self.slots[VALUES_SLOT] .take() .vortex_expect("RunEndArray values slot"), + offset: self.offset, } } } @@ -524,7 +495,7 @@ impl ValidityVTable for RunEnd { Validity::NonNullable | Validity::AllValid => Validity::AllValid, Validity::AllInvalid => Validity::AllInvalid, Validity::Array(values_validity) => Validity::Array(unsafe { - RunEndData::new_unchecked( + RunEnd::new_unchecked( array.ends().clone(), values_validity, array.offset(), diff --git a/encodings/runend/src/arrow.rs b/encodings/runend/src/arrow.rs index 82eb837f48f..8d9e674f778 100644 --- a/encodings/runend/src/arrow.rs +++ b/encodings/runend/src/arrow.rs @@ -30,7 +30,12 @@ where .reinterpret_cast(R::Native::PTYPE.to_unsigned()); let values = ArrayRef::from_arrow(array.values().as_ref(), nullable)?; - let ends_array = ends.into_array(); + let ends_array = PrimitiveArray::from_buffer_handle( + ends.buffer_handle().clone(), + ends.ptype(), + ends.validity(), + ) + .into_array(); let (ends_slice, values_slice) = if offset == 0 && len == array.run_ends().max_value() { (ends_array, values) } else { @@ -46,7 +51,7 @@ where ) }; - // SAFETY: arrow-rs enforces the RunEndArray invariants, we inherit their guarantees + // SAFETY: arrow-rs enforces the RunEndArray invariants, we inherit their guarantees. Ok(unsafe { RunEndData::new_unchecked(ends_slice, values_slice, offset, len) }) } } @@ -56,12 +61,14 @@ mod tests { use std::sync::Arc; use std::sync::LazyLock; + use arrow_array::Array as _; use arrow_array::Float64Array; use arrow_array::Int32Array; use arrow_array::Int64Array; use arrow_array::RunArray; use arrow_array::types::Int32Type; use arrow_array::types::Int64Type; + use arrow_array::types::RunEndIndexType; use arrow_schema::DataType; use arrow_schema::Field; use rstest::rstest; @@ -72,6 +79,7 @@ mod tests { use vortex_array::arrow::FromArrowArray; use vortex_array::assert_arrays_eq; use vortex_array::dtype::DType; + use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::session::ArraySession; @@ -79,11 +87,28 @@ mod tests { use vortex_error::VortexResult; use vortex_session::VortexSession; + use crate::RunEnd; use crate::RunEndData; static SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); + fn decode_run_array( + array: &RunArray, + nullable: bool, + ) -> VortexResult + where + R::Native: NativePType, + { + let data = RunEndData::from_arrow(array, nullable)?; + RunEnd::try_new_offset_length( + data.ends().clone(), + data.values().clone(), + data.offset(), + array.len(), + ) + } + #[test] fn test_arrow_run_array_to_vortex() -> VortexResult<()> { // Create an Arrow RunArray with UInt32 run ends and Int32 values @@ -94,7 +119,7 @@ mod tests { let arrow_run_array = RunArray::::try_new(&run_ends, &values).unwrap(); // Convert to Vortex - let vortex_array = RunEndData::from_arrow(&arrow_run_array, false)?; + let vortex_array = decode_run_array(&arrow_run_array, false)?; assert_arrays_eq!( vortex_array.into_array(), @@ -111,7 +136,7 @@ mod tests { let arrow_run_array = RunArray::::try_new(&run_ends, &values).unwrap(); // Convert to Vortex with nullable=true - let vortex_array = RunEndData::from_arrow(&arrow_run_array, true)?; + let vortex_array = decode_run_array(&arrow_run_array, true)?; assert_arrays_eq!( vortex_array.into_array(), @@ -135,7 +160,7 @@ mod tests { let arrow_run_array = RunArray::::try_new(&run_ends, &values).unwrap(); // Convert to Vortex - let vortex_array = RunEndData::from_arrow(&arrow_run_array, false)?; + let vortex_array = decode_run_array(&arrow_run_array, false)?; assert_arrays_eq!(vortex_array, buffer![1.5f64, 2.5, 2.5, 3.5].into_array()); Ok(()) @@ -155,7 +180,7 @@ mod tests { let sliced_array = arrow_run_array.slice(1, 6); // Convert the sliced array to Vortex - let vortex_array = RunEndData::from_arrow(&sliced_array, false)?; + let vortex_array = decode_run_array(&sliced_array, false)?; assert_arrays_eq!( vortex_array, buffer![100, 200, 200, 200, 300, 300].into_array() @@ -178,7 +203,7 @@ mod tests { let sliced_array = arrow_run_array.slice(4, 6); // Convert to Vortex with nullable=true - let vortex_array = RunEndData::from_arrow(&sliced_array, true)?; + let vortex_array = decode_run_array(&sliced_array, true)?; assert_arrays_eq!( vortex_array, @@ -209,7 +234,7 @@ mod tests { let sliced_array = arrow_run_array.slice(4, 0); // Convert to Vortex with nullable=true - let vortex_array = RunEndData::from_arrow(&sliced_array, true)?; + let vortex_array = decode_run_array(&sliced_array, true)?; // Verify properties assert_eq!(vortex_array.len(), 0); @@ -240,7 +265,7 @@ mod tests { &Int32Array::from(vec![3i32, 5, 8]), &Int32Array::from(vec![10, 20, 30]), )?; - let vortex_array = RunEndData::from_arrow(&original, false)?; + let vortex_array = decode_run_array(&original, false)?; let target = ree_type(DataType::Int32, DataType::Int32); let result = execute(vortex_array.into_array(), &target)?; @@ -273,8 +298,7 @@ mod tests { #[case] expected_ends: &[i32], #[case] expected_values: &[i32], ) -> VortexResult<()> { - let array = - RunEndData::encode(PrimitiveArray::from_iter(input.iter().copied()).into_array())?; + let array = RunEnd::encode(PrimitiveArray::from_iter(input.iter().copied()).into_array())?; let sliced = array.into_array().slice(slice_range.clone())?; let target = ree_type(DataType::Int32, DataType::Int32); let result = execute(sliced, &target)?; diff --git a/encodings/runend/src/compute/cast.rs b/encodings/runend/src/compute/cast.rs index a5fec75d802..d9ecb3a0baf 100644 --- a/encodings/runend/src/compute/cast.rs +++ b/encodings/runend/src/compute/cast.rs @@ -10,7 +10,6 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::RunEnd; -use crate::RunEndData; impl CastReduce for RunEnd { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // Cast the values array to the target type @@ -19,7 +18,7 @@ impl CastReduce for RunEnd { // SAFETY: casting does not affect the ends being valid unsafe { Ok(Some( - RunEndData::new_unchecked( + RunEnd::new_unchecked( array.ends().clone(), casted_values, array.offset(), diff --git a/encodings/runend/src/compute/fill_null.rs b/encodings/runend/src/compute/fill_null.rs index bb138e406d6..abe0ef44f09 100644 --- a/encodings/runend/src/compute/fill_null.rs +++ b/encodings/runend/src/compute/fill_null.rs @@ -10,7 +10,6 @@ use vortex_array::scalar_fn::fns::fill_null::FillNullReduce; use vortex_error::VortexResult; use crate::RunEnd; -use crate::RunEndData; impl FillNullReduce for RunEnd { fn fill_null( @@ -21,7 +20,7 @@ impl FillNullReduce for RunEnd { // SAFETY: modifying values only, does not affect ends Ok(Some( unsafe { - RunEndData::new_unchecked( + RunEnd::new_unchecked( array.ends().clone(), new_values, array.offset(), diff --git a/encodings/runend/src/compute/filter.rs b/encodings/runend/src/compute/filter.rs index 92c3ef40b8e..e0ef5382d40 100644 --- a/encodings/runend/src/compute/filter.rs +++ b/encodings/runend/src/compute/filter.rs @@ -21,7 +21,6 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::RunEnd; -use crate::RunEndData; use crate::compute::take::take_indices_unchecked; const FILTER_TAKE_THRESHOLD: f64 = 0.1; @@ -59,7 +58,7 @@ impl FilterKernel for RunEnd { // SAFETY: guaranteed by implementation of filter_run_end_primitive unsafe { Ok(Some( - RunEndData::new_unchecked( + RunEnd::new_unchecked( run_ends.into_array(), values, 0, diff --git a/encodings/runend/src/kernel.rs b/encodings/runend/src/kernel.rs index 2ca1586bb51..97f17c17857 100644 --- a/encodings/runend/src/kernel.rs +++ b/encodings/runend/src/kernel.rs @@ -17,7 +17,6 @@ use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor; use vortex_error::VortexResult; use crate::RunEnd; -use crate::RunEndData; use crate::compute::take_from::RunEndTakeFrom; pub(super) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ @@ -45,11 +44,11 @@ impl ExecuteParentKernel for RunEndSliceKernel { _child_idx: usize, _ctx: &mut ExecutionCtx, ) -> VortexResult> { - slice(&array, parent.slice_range().clone()).map(Some) + slice(array, parent.slice_range().clone()).map(Some) } } -fn slice(array: &RunEndData, range: Range) -> VortexResult { +fn slice(array: ArrayView<'_, RunEnd>, range: Range) -> VortexResult { let new_length = range.len(); let slice_begin = array.find_physical_index(range.start)?; @@ -63,7 +62,7 @@ fn slice(array: &RunEndData, range: Range) -> VortexResult { // SAFETY: we maintain the ends invariant in our slice implementation Ok(unsafe { - RunEndData::new_unchecked( + RunEnd::new_unchecked( array.ends().slice(slice_begin..slice_end)?, array.values().slice(slice_begin..slice_end)?, range.start + array.offset(), diff --git a/encodings/runend/src/lib.rs b/encodings/runend/src/lib.rs index 13ca4f2e596..71c706049b3 100644 --- a/encodings/runend/src/lib.rs +++ b/encodings/runend/src/lib.rs @@ -58,7 +58,7 @@ pub fn initialize(session: &mut VortexSession) { #[cfg(test)] mod tests { - use vortex_array::ProstMetadata; + use prost::Message; use vortex_array::dtype::PType; use vortex_array::test_harness::check_metadata; @@ -69,11 +69,12 @@ mod tests { fn test_runend_metadata() { check_metadata( "runend.metadata", - ProstMetadata(RunEndMetadata { + &RunEndMetadata { ends_ptype: PType::U64 as i32, num_runs: u64::MAX, offset: u64::MAX, - }), + } + .encode_to_vec(), ); } } diff --git a/encodings/runend/src/rules.rs b/encodings/runend/src/rules.rs index 4ad3db969c3..2c8a6607156 100644 --- a/encodings/runend/src/rules.rs +++ b/encodings/runend/src/rules.rs @@ -17,7 +17,6 @@ use vortex_array::scalar_fn::fns::fill_null::FillNullReduceAdaptor; use vortex_error::VortexResult; use crate::RunEnd; -use crate::RunEndData; pub(super) const RULES: ParentRuleSet = ParentRuleSet::new(&[ // CastReduceAdaptor must come before RunEndScalarFnRule so that cast operations are executed @@ -81,7 +80,7 @@ impl ArrayParentReduceRule for RunEndScalarFnRule { Ok(Some( unsafe { - RunEndData::new_unchecked( + RunEnd::new_unchecked( run_end.ends().clone(), new_values, run_end.offset(), diff --git a/encodings/sequence/public-api.lock b/encodings/sequence/public-api.lock index 95ccfef438c..f9b2995ced4 100644 --- a/encodings/sequence/public-api.lock +++ b/encodings/sequence/public-api.lock @@ -22,8 +22,6 @@ impl vortex_array::array::vtable::VTable for vortex_sequence::Sequence pub type vortex_sequence::Sequence::ArrayData = vortex_sequence::SequenceData -pub type vortex_sequence::Sequence::Metadata = vortex_sequence::array::SequenceMetadata - pub type vortex_sequence::Sequence::OperationsVTable = vortex_sequence::Sequence pub type vortex_sequence::Sequence::ValidityVTable = vortex_sequence::Sequence @@ -36,11 +34,7 @@ pub fn vortex_sequence::Sequence::buffer(_array: vortex_array::array::view::Arra pub fn vortex_sequence::Sequence::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_sequence::Sequence::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_sequence::Sequence::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_sequence::Sequence::dtype(array: &vortex_sequence::SequenceData) -> &vortex_array::dtype::DType +pub fn vortex_sequence::Sequence::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_sequence::Sequence::execute(array: vortex_array::array::typed::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -48,23 +42,17 @@ pub fn vortex_sequence::Sequence::execute_parent(array: vortex_array::array::vie pub fn vortex_sequence::Sequence::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_sequence::Sequence::len(array: &vortex_sequence::SequenceData) -> usize - -pub fn vortex_sequence::Sequence::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_sequence::Sequence::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_sequence::Sequence::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_sequence::Sequence::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_sequence::Sequence::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_sequence::Sequence::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_sequence::Sequence::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_sequence::Sequence::stats(array: &vortex_sequence::SequenceData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_sequence::Sequence::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_sequence::Sequence::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_sequence::Sequence::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -100,57 +88,35 @@ impl vortex_array::scalar_fn::fns::list_contains::kernel::ListContainsElementRed pub fn vortex_sequence::Sequence::list_contains(list: &vortex_array::array::erased::ArrayRef, element: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult> -pub struct vortex_sequence::SequenceArrayParts - -pub vortex_sequence::SequenceArrayParts::base: vortex_array::scalar::typed_view::primitive::pvalue::PValue - -pub vortex_sequence::SequenceArrayParts::len: usize - -pub vortex_sequence::SequenceArrayParts::multiplier: vortex_array::scalar::typed_view::primitive::pvalue::PValue - -pub vortex_sequence::SequenceArrayParts::nullability: vortex_array::dtype::nullability::Nullability - -pub vortex_sequence::SequenceArrayParts::ptype: vortex_array::dtype::ptype::PType - pub struct vortex_sequence::SequenceData impl vortex_sequence::SequenceData pub fn vortex_sequence::SequenceData::base(&self) -> vortex_array::scalar::typed_view::primitive::pvalue::PValue -pub fn vortex_sequence::SequenceData::dtype(&self) -> &vortex_array::dtype::DType - -pub fn vortex_sequence::SequenceData::into_parts(self) -> vortex_sequence::SequenceArrayParts - -pub fn vortex_sequence::SequenceData::is_empty(&self) -> bool - -pub fn vortex_sequence::SequenceData::last(&self) -> vortex_array::scalar::typed_view::primitive::pvalue::PValue - -pub fn vortex_sequence::SequenceData::len(&self) -> usize +pub fn vortex_sequence::SequenceData::into_parts(self) -> vortex_sequence::SequenceDataParts pub fn vortex_sequence::SequenceData::multiplier(&self) -> vortex_array::scalar::typed_view::primitive::pvalue::PValue pub fn vortex_sequence::SequenceData::ptype(&self) -> vortex_array::dtype::ptype::PType -pub fn vortex_sequence::SequenceData::try_new(base: vortex_array::scalar::typed_view::primitive::pvalue::PValue, multiplier: vortex_array::scalar::typed_view::primitive::pvalue::PValue, ptype: vortex_array::dtype::ptype::PType, nullability: vortex_array::dtype::nullability::Nullability, length: usize) -> vortex_error::VortexResult - -pub fn vortex_sequence::SequenceData::try_new_typed>(base: T, multiplier: T, nullability: vortex_array::dtype::nullability::Nullability, length: usize) -> vortex_error::VortexResult +pub fn vortex_sequence::SequenceData::validate(base: vortex_array::scalar::typed_view::primitive::pvalue::PValue, multiplier: vortex_array::scalar::typed_view::primitive::pvalue::PValue, dtype: &vortex_array::dtype::DType, length: usize) -> vortex_error::VortexResult<()> impl core::clone::Clone for vortex_sequence::SequenceData pub fn vortex_sequence::SequenceData::clone(&self) -> vortex_sequence::SequenceData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_sequence::SequenceData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_sequence::SequenceData pub fn vortex_sequence::SequenceData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_sequence::SequenceData +pub struct vortex_sequence::SequenceDataParts + +pub vortex_sequence::SequenceDataParts::base: vortex_array::scalar::typed_view::primitive::pvalue::PValue + +pub vortex_sequence::SequenceDataParts::multiplier: vortex_array::scalar::typed_view::primitive::pvalue::PValue -pub fn vortex_sequence::SequenceData::into_array(self) -> vortex_array::array::erased::ArrayRef +pub vortex_sequence::SequenceDataParts::ptype: vortex_array::dtype::ptype::PType pub fn vortex_sequence::initialize(session: &mut vortex_session::VortexSession) diff --git a/encodings/sequence/src/array.rs b/encodings/sequence/src/array.rs index 4b38b5f9124..a3d50a420e1 100644 --- a/encodings/sequence/src/array.rs +++ b/encodings/sequence/src/array.rs @@ -4,16 +4,15 @@ use std::hash::Hash; use num_traits::cast::FromPrimitive; +use prost::Message; use vortex_array::Array; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; -use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::Precision; -use vortex_array::ProstMetadata; -use vortex_array::SerializeMetadata; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; @@ -29,7 +28,6 @@ use vortex_array::scalar::PValue; use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::stats::StatsSet; use vortex_array::validity::Validity; use vortex_array::vtable; @@ -50,29 +48,14 @@ use crate::rules::RULES; vtable!(Sequence, Sequence, SequenceData); -#[derive(Debug, Clone, Copy)] -pub struct SequenceMetadata { - base: PValue, - multiplier: PValue, -} - #[derive(Clone, prost::Message)] -pub struct ProstSequenceMetadata { +pub struct SequenceMetadata { #[prost(message, tag = "1")] base: Option, #[prost(message, tag = "2")] multiplier: Option, } -/// Components of [`SequenceArray`]. -pub struct SequenceArrayParts { - pub base: PValue, - pub multiplier: PValue, - pub len: usize, - pub ptype: PType, - pub nullability: Nullability, -} - pub(super) const SLOT_NAMES: [&str; 0] = []; #[derive(Clone, Debug)] @@ -80,14 +63,17 @@ pub(super) const SLOT_NAMES: [&str; 0] = []; pub struct SequenceData { base: PValue, multiplier: PValue, - dtype: DType, - pub(crate) len: usize, pub(super) slots: Vec>, - stats_set: ArrayStats, +} + +pub struct SequenceDataParts { + pub base: PValue, + pub multiplier: PValue, + pub ptype: PType, } impl SequenceData { - pub fn try_new_typed>( + pub(crate) fn try_new_typed>( base: T, multiplier: T, nullability: Nullability, @@ -103,99 +89,70 @@ impl SequenceData { } /// Constructs a sequence array using two integer values (with the same ptype). - pub fn try_new( + pub(crate) fn try_new( base: PValue, multiplier: PValue, ptype: PType, nullability: Nullability, length: usize, ) -> VortexResult { + let dtype = DType::Primitive(ptype, nullability); + Self::validate(base, multiplier, &dtype, length)?; + let (base, multiplier) = Self::normalize(base, multiplier, ptype)?; + + Ok(unsafe { Self::new_unchecked(base, multiplier) }) + } + + pub fn validate( + base: PValue, + multiplier: PValue, + dtype: &DType, + length: usize, + ) -> VortexResult<()> { + let DType::Primitive(ptype, _) = dtype else { + vortex_bail!("only primitive dtypes are supported in SequenceArray currently"); + }; + if !ptype.is_int() { vortex_bail!("only integer ptype are supported in SequenceArray currently") } - Self::try_last(base, multiplier, ptype, length).map_err(|e| { + vortex_ensure!(length > 0, "SequenceArray length must be greater than zero"); + Self::try_last(base, multiplier, *ptype, length).map_err(|e| { e.with_context(format!( "final value not expressible, base = {base:?}, multiplier = {multiplier:?}, len = {length} ", )) })?; - // SAFETY: we just validated that `ptype` is an integer and that the final - // element is representable via `try_last`. - Ok(unsafe { Self::new_unchecked(base, multiplier, ptype, nullability, length) }) + Ok(()) } - /// Constructs a [`SequenceArray`] without validating that the `ptype` is an integer - /// type or that the final element is representable. + fn normalize(base: PValue, multiplier: PValue, ptype: PType) -> VortexResult<(PValue, PValue)> { + match_each_integer_ptype!(ptype, |P| { + Ok(( + PValue::from(base.cast::

()?), + PValue::from(multiplier.cast::

()?), + )) + }) + } + + /// Constructs a [`SequenceArray`] payload without validation. /// /// # Safety /// /// The caller must ensure that: - /// - `ptype` is an integer type (i.e., `ptype.is_int()` returns `true`). - /// - `base + (length - 1) * multiplier` does not overflow the range of `ptype`. - /// - /// Violating the first invariant will cause a panic. Violating the second will - /// cause silent wraparound when materializing elements, producing incorrect values. - pub(crate) unsafe fn new_unchecked( - base: PValue, - multiplier: PValue, - ptype: PType, - nullability: Nullability, - length: usize, - ) -> Self { - let dtype = DType::Primitive(ptype, nullability); - - // A sequence A[i] = base + i * multiplier is sorted iff multiplier >= 0, - // and strictly sorted iff multiplier > 0. - - let (is_sorted, is_strict_sorted) = match_each_pvalue!( - multiplier, - uint: |v| { (true, v> 0) }, - int: |v| { (v >= 0, v > 0) }, - float: |_v| { unreachable!("float multiplier not supported") } - ); - - // SAFETY: we don't have duplicate stats - let stats_set = unsafe { - StatsSet::new_unchecked(vec![ - (Stat::IsSorted, StatPrecision::Exact(is_sorted.into())), - ( - Stat::IsStrictSorted, - StatPrecision::Exact(is_strict_sorted.into()), - ), - ]) - }; - + /// - `base` and `multiplier` are both normalized to the same integer `ptype`. + /// - they are logically compatible with the outer dtype and len. + pub(crate) unsafe fn new_unchecked(base: PValue, multiplier: PValue) -> Self { Self { base, multiplier, - dtype, - len: length, slots: vec![], - stats_set: ArrayStats::from(stats_set), } } - /// Returns the length of the array. - #[inline] - pub fn len(&self) -> usize { - self.len - } - - /// Returns whether the array is empty. - #[inline] - pub fn is_empty(&self) -> bool { - self.len == 0 - } - - /// Returns the logical data type of the array. - #[inline] - pub fn dtype(&self) -> &DType { - &self.dtype - } - pub fn ptype(&self) -> PType { - self.dtype.as_ptype() + self.base.ptype() } pub fn base(&self) -> PValue { @@ -206,6 +163,14 @@ impl SequenceData { self.multiplier } + pub fn into_parts(self) -> SequenceDataParts { + SequenceDataParts { + base: self.base, + multiplier: self.multiplier, + ptype: self.base.ptype(), + } + } + pub(crate) fn try_last( base: PValue, multiplier: PValue, @@ -227,8 +192,6 @@ impl SequenceData { } pub(crate) fn index_value(&self, idx: usize) -> PValue { - assert!(idx < self.len, "index_value({idx}): index out of bounds"); - match_each_native_ptype!(self.ptype(), |P| { let base = self.base.cast::

().vortex_expect("must be able to cast"); let multiplier = self @@ -240,49 +203,20 @@ impl SequenceData { PValue::from(value) }) } - - /// Returns the validated final value of a sequence array - pub fn last(&self) -> PValue { - Self::try_last(self.base, self.multiplier, self.ptype(), self.len) - .vortex_expect("validated array") - } - - pub fn into_parts(self) -> SequenceArrayParts { - SequenceArrayParts { - base: self.base, - multiplier: self.multiplier, - len: self.len, - ptype: self.dtype.as_ptype(), - nullability: self.dtype.nullability(), - } - } } impl VTable for Sequence { type ArrayData = SequenceData; - type Metadata = SequenceMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Sequence - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &SequenceData) -> usize { - array.len - } - - fn dtype(array: &SequenceData) -> &DType { - &array.dtype - } - - fn stats(array: &SequenceData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + SequenceData::validate(data.base, data.multiplier, dtype, len) } fn array_hash( @@ -310,37 +244,41 @@ impl VTable for Sequence { vortex_panic!("SequenceArray buffer_name index {idx} out of bounds") } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(SequenceMetadata { - base: array.base(), - multiplier: array.multiplier(), - }) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - let prost = ProstMetadata(ProstSequenceMetadata { - base: Some((&metadata.base).into()), - multiplier: Some((&metadata.multiplier).into()), - }); + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + let metadata = SequenceMetadata { + base: Some((&array.base()).into()), + multiplier: Some((&array.multiplier()).into()), + }; - Ok(Some(prost.serialize())) + Ok(Some(metadata.encode_to_vec())) } fn deserialize( - bytes: &[u8], + &self, dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], + len: usize, + metadata: &[u8], + buffers: &[BufferHandle], + children: &dyn ArrayChildren, session: &VortexSession, - ) -> VortexResult { - let prost = - as DeserializeMetadata>::deserialize(bytes)?; + ) -> VortexResult { + vortex_ensure!( + buffers.is_empty(), + "SequenceArray expects 0 buffers, got {}", + buffers.len() + ); + vortex_ensure!( + children.is_empty(), + "SequenceArray expects 0 children, got {}", + children.len() + ); + let metadata = SequenceMetadata::decode(metadata)?; let ptype = dtype.as_ptype(); // We go via Scalar to validate that the value is valid for the ptype. let base = Scalar::from_proto_value( - prost + metadata .base .as_ref() .ok_or_else(|| vortex_err!("base required"))?, @@ -352,7 +290,7 @@ impl VTable for Sequence { .vortex_expect("sequence array base should be a non-nullable primitive"); let multiplier = Scalar::from_proto_value( - prost + metadata .multiplier .as_ref() .ok_or_else(|| vortex_err!("multiplier required"))?, @@ -363,23 +301,7 @@ impl VTable for Sequence { .pvalue() .vortex_expect("sequence array multiplier should be a non-nullable primitive"); - Ok(SequenceMetadata { base, multiplier }) - } - - fn build( - dtype: &DType, - len: usize, - metadata: &Self::Metadata, - _buffers: &[BufferHandle], - _children: &dyn ArrayChildren, - ) -> VortexResult { - SequenceData::try_new( - metadata.base, - metadata.multiplier, - dtype.as_ptype(), - dtype.nullability(), - len, - ) + SequenceData::try_new(base, multiplier, ptype, dtype.nullability(), len) } fn slots(array: ArrayView<'_, Self>) -> &[Option] { @@ -447,6 +369,49 @@ pub struct Sequence; impl Sequence { pub const ID: ArrayId = ArrayId::new_ref("vortex.sequence"); + fn stats(multiplier: PValue) -> StatsSet { + // A sequence A[i] = base + i * multiplier is sorted iff multiplier >= 0, + // and strictly sorted iff multiplier > 0. + let (is_sorted, is_strict_sorted) = match_each_pvalue!( + multiplier, + uint: |v| { (true, v > 0) }, + int: |v| { (v >= 0, v > 0) }, + float: |_v| { unreachable!("float multiplier not supported") } + ); + + // SAFETY: we don't have duplicate stats. + unsafe { + StatsSet::new_unchecked(vec![ + (Stat::IsSorted, StatPrecision::Exact(is_sorted.into())), + ( + Stat::IsStrictSorted, + StatPrecision::Exact(is_strict_sorted.into()), + ), + ]) + } + } + + /// Construct a new [`SequenceArray`] from pre-validated parts. + /// + /// # Safety + /// + /// Caller must ensure the sequence is logically compatible with the provided dtype and len. + pub(crate) unsafe fn new_unchecked( + base: PValue, + multiplier: PValue, + ptype: PType, + nullability: Nullability, + length: usize, + ) -> SequenceArray { + let dtype = DType::Primitive(ptype, nullability); + let (base, multiplier) = SequenceData::normalize(base, multiplier, ptype) + .vortex_expect("SequenceArray parts must be normalized to the target ptype"); + let stats = Self::stats(multiplier); + let data = unsafe { SequenceData::new_unchecked(base, multiplier) }; + unsafe { Array::from_parts_unchecked(ArrayParts::new(Sequence, dtype, length, data)) } + .with_stats_set(stats) + } + /// Construct a new [`SequenceArray`] from its components. pub fn try_new( base: PValue, @@ -455,13 +420,13 @@ impl Sequence { nullability: Nullability, length: usize, ) -> VortexResult { - Array::try_from_data(SequenceData::try_new( - base, - multiplier, - ptype, - nullability, - length, - )?) + let dtype = DType::Primitive(ptype, nullability); + let data = SequenceData::try_new(base, multiplier, ptype, nullability, length)?; + let stats = Self::stats(data.multiplier()); + Ok( + unsafe { Array::from_parts_unchecked(ArrayParts::new(Sequence, dtype, length, data)) } + .with_stats_set(stats), + ) } /// Construct a new typed [`SequenceArray`] from base/multiplier values. @@ -471,12 +436,14 @@ impl Sequence { nullability: Nullability, length: usize, ) -> VortexResult { - Array::try_from_data(SequenceData::try_new_typed( - base, - multiplier, - nullability, - length, - )?) + let ptype = T::PTYPE; + let dtype = DType::Primitive(ptype, nullability); + let data = SequenceData::try_new_typed(base, multiplier, nullability, length)?; + let stats = Self::stats(data.multiplier()); + Ok( + unsafe { Array::from_parts_unchecked(ArrayParts::new(Sequence, dtype, length, data)) } + .with_stats_set(stats), + ) } } diff --git a/encodings/sequence/src/compress.rs b/encodings/sequence/src/compress.rs index b073ace8dc2..6321aa3df78 100644 --- a/encodings/sequence/src/compress.rs +++ b/encodings/sequence/src/compress.rs @@ -18,9 +18,10 @@ use vortex_buffer::BufferMut; use vortex_buffer::trusted_len::TrustedLen; use vortex_error::VortexResult; +use crate::Sequence; use crate::SequenceArray; -/// An iterator that yields `base, base + step, base + 2*step, ...` via repeated addition. use crate::SequenceData; +/// An iterator that yields `base, base + step, base + 2*step, ...` via repeated addition. struct SequenceIter { acc: T, step: T, @@ -112,7 +113,7 @@ fn encode_primitive_array + CheckedAdd + CheckedSu ) -> VortexResult> { if slice.len() == 1 { // The multiplier here can be any value, zero is chosen - return SequenceData::try_new_typed(slice[0], P::zero(), nullability, 1) + return Sequence::try_new_typed(slice[0], P::zero(), nullability, 1) .map(|a| Some(a.into_array())); } let base = slice[0]; @@ -133,7 +134,7 @@ fn encode_primitive_array + CheckedAdd + CheckedSu .windows(2) .all(|w| Some(w[1]) == w[0].checked_add(&multiplier)) .then_some( - SequenceData::try_new_typed(base, multiplier, nullability, slice.len()) + Sequence::try_new_typed(base, multiplier, nullability, slice.len()) .map(|a| a.into_array()), ) .transpose() diff --git a/encodings/sequence/src/compute/cast.rs b/encodings/sequence/src/compute/cast.rs index 54f3393f16e..1eb9ddf47d8 100644 --- a/encodings/sequence/src/compute/cast.rs +++ b/encodings/sequence/src/compute/cast.rs @@ -13,7 +13,6 @@ use vortex_error::VortexResult; use vortex_error::vortex_err; use crate::Sequence; -use crate::SequenceData; impl CastReduce for Sequence { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // SequenceArray represents arithmetic sequences (base + i * multiplier) which @@ -32,7 +31,7 @@ impl CastReduce for Sequence { // For SequenceArray, we can just create a new one with the same parameters // but different nullability return Ok(Some( - SequenceData::try_new( + Sequence::try_new( array.base(), array.multiplier(), *target_ptype, @@ -71,7 +70,7 @@ impl CastReduce for Sequence { .ok_or_else(|| vortex_err!("Cast resulted in null multiplier value"))?; return Ok(Some( - SequenceData::try_new( + Sequence::try_new( new_base, new_multiplier, *target_ptype, diff --git a/encodings/sequence/src/compute/min_max.rs b/encodings/sequence/src/compute/min_max.rs index 17441bbeb4b..2ff8b2abd58 100644 --- a/encodings/sequence/src/compute/min_max.rs +++ b/encodings/sequence/src/compute/min_max.rs @@ -16,6 +16,7 @@ use vortex_array::scalar::ScalarValue; use vortex_error::VortexResult; use crate::Sequence; +use crate::SequenceData; /// Sequence-specific min/max kernel. /// @@ -47,7 +48,7 @@ impl DynAggregateKernel for SequenceMinMaxKernel { } let base = seq.base(); - let last = seq.last(); + let last = SequenceData::try_last(base, seq.multiplier(), seq.ptype(), seq.len())?; // Determine min and max based on multiplier direction. // For unsigned types, multiplier is always >= 0. diff --git a/encodings/sequence/src/compute/slice.rs b/encodings/sequence/src/compute/slice.rs index 3eb79f0b23b..c2b64b68ef4 100644 --- a/encodings/sequence/src/compute/slice.rs +++ b/encodings/sequence/src/compute/slice.rs @@ -10,14 +10,13 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::Sequence; -use crate::SequenceData; impl SliceReduce for Sequence { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { // SAFETY: this is a slice of an already-validated `SequenceArray`, so this is still valid. Ok(Some( unsafe { - SequenceData::new_unchecked( + Sequence::new_unchecked( array.index_value(range.start), array.multiplier(), array.ptype(), diff --git a/encodings/sequence/src/lib.rs b/encodings/sequence/src/lib.rs index 958bbf6352f..a9505d8178a 100644 --- a/encodings/sequence/src/lib.rs +++ b/encodings/sequence/src/lib.rs @@ -13,8 +13,8 @@ pub use array::Sequence; /// Represents the equation A\[i\] = a * i + b. /// This can be used for compression, fast comparisons and also for row ids. pub use array::SequenceArray; -pub use array::SequenceArrayParts; pub use array::SequenceData; +pub use array::SequenceDataParts; pub use compress::sequence_encode; use vortex_array::aggregate_fn::AggregateFnVTable; use vortex_array::aggregate_fn::fns::is_sorted::IsSorted; diff --git a/encodings/sparse/public-api.lock b/encodings/sparse/public-api.lock index 71b42ab8dbd..b937d3df263 100644 --- a/encodings/sparse/public-api.lock +++ b/encodings/sparse/public-api.lock @@ -1,25 +1,5 @@ pub mod vortex_sparse -#[repr(C)] pub struct vortex_sparse::ProstPatchesMetadata - -impl core::clone::Clone for vortex_sparse::ProstPatchesMetadata - -pub fn vortex_sparse::ProstPatchesMetadata::clone(&self) -> vortex_sparse::ProstPatchesMetadata - -impl core::default::Default for vortex_sparse::ProstPatchesMetadata - -pub fn vortex_sparse::ProstPatchesMetadata::default() -> Self - -impl core::fmt::Debug for vortex_sparse::ProstPatchesMetadata - -pub fn vortex_sparse::ProstPatchesMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -impl prost::message::Message for vortex_sparse::ProstPatchesMetadata - -pub fn vortex_sparse::ProstPatchesMetadata::clear(&mut self) - -pub fn vortex_sparse::ProstPatchesMetadata::encoded_len(&self) -> usize - pub struct vortex_sparse::Sparse impl vortex_sparse::Sparse @@ -30,6 +10,8 @@ pub fn vortex_sparse::Sparse::encode(array: &vortex_array::array::erased::ArrayR pub fn vortex_sparse::Sparse::try_new(indices: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, len: usize, fill_value: vortex_array::scalar::Scalar) -> vortex_error::VortexResult +pub fn vortex_sparse::Sparse::try_new_from_patches(patches: vortex_array::patches::Patches, fill_value: vortex_array::scalar::Scalar) -> vortex_error::VortexResult + impl core::clone::Clone for vortex_sparse::Sparse pub fn vortex_sparse::Sparse::clone(&self) -> vortex_sparse::Sparse @@ -42,8 +24,6 @@ impl vortex_array::array::vtable::VTable for vortex_sparse::Sparse pub type vortex_sparse::Sparse::ArrayData = vortex_sparse::SparseData -pub type vortex_sparse::Sparse::Metadata = vortex_sparse::SparseMetadata - pub type vortex_sparse::Sparse::OperationsVTable = vortex_sparse::Sparse pub type vortex_sparse::Sparse::ValidityVTable = vortex_sparse::Sparse @@ -56,11 +36,7 @@ pub fn vortex_sparse::Sparse::buffer(array: vortex_array::array::view::ArrayView pub fn vortex_sparse::Sparse::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_sparse::Sparse::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_sparse::Sparse::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_sparse::Sparse::dtype(array: &vortex_sparse::SparseData) -> &vortex_array::dtype::DType +pub fn vortex_sparse::Sparse::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_sparse::Sparse::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -68,23 +44,17 @@ pub fn vortex_sparse::Sparse::execute_parent(array: vortex_array::array::view::A pub fn vortex_sparse::Sparse::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_sparse::Sparse::len(array: &vortex_sparse::SparseData) -> usize - -pub fn vortex_sparse::Sparse::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_sparse::Sparse::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_sparse::Sparse::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_sparse::Sparse::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_sparse::Sparse::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_sparse::Sparse::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_sparse::Sparse::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_sparse::Sparse::stats(array: &vortex_sparse::SparseData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_sparse::Sparse::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_sparse::Sparse::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_sparse::Sparse::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -138,26 +108,34 @@ pub fn vortex_sparse::SparseData::try_new(indices: vortex_array::array::erased:: pub fn vortex_sparse::SparseData::try_new_from_patches(patches: vortex_array::patches::Patches, fill_value: vortex_array::scalar::Scalar) -> vortex_error::VortexResult +pub fn vortex_sparse::SparseData::validate(patches: &vortex_array::patches::Patches, fill_value: &vortex_array::scalar::Scalar, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> + impl core::clone::Clone for vortex_sparse::SparseData pub fn vortex_sparse::SparseData::clone(&self) -> vortex_sparse::SparseData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_sparse::SparseData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_sparse::SparseData pub fn vortex_sparse::SparseData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_sparse::SparseData +#[repr(C)] pub struct vortex_sparse::SparseMetadata + +impl core::clone::Clone for vortex_sparse::SparseMetadata + +pub fn vortex_sparse::SparseMetadata::clone(&self) -> vortex_sparse::SparseMetadata -pub fn vortex_sparse::SparseData::into_array(self) -> vortex_array::array::erased::ArrayRef +impl core::default::Default for vortex_sparse::SparseMetadata -pub struct vortex_sparse::SparseMetadata +pub fn vortex_sparse::SparseMetadata::default() -> Self impl core::fmt::Debug for vortex_sparse::SparseMetadata pub fn vortex_sparse::SparseMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl prost::message::Message for vortex_sparse::SparseMetadata + +pub fn vortex_sparse::SparseMetadata::clear(&mut self) + +pub fn vortex_sparse::SparseMetadata::encoded_len(&self) -> usize + pub type vortex_sparse::SparseArray = vortex_array::array::typed::Array diff --git a/encodings/sparse/src/canonical.rs b/encodings/sparse/src/canonical.rs index a3cb566dddb..c0da00a14d7 100644 --- a/encodings/sparse/src/canonical.rs +++ b/encodings/sparse/src/canonical.rs @@ -5,10 +5,13 @@ use std::sync::Arc; use itertools::Itertools; use num_traits::NumCast; +use vortex_array::Array; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::BoolArray; +use vortex_array::arrays::Decimal; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::ListViewArray; use vortex_array::arrays::NullArray; @@ -52,8 +55,8 @@ use vortex_error::vortex_bail; use vortex_error::vortex_panic; use crate::ConstantArray; +use crate::Sparse; use crate::SparseArray; -use crate::SparseData; pub(super) fn execute_sparse( array: &SparseArray, ctx: &mut ExecutionCtx, @@ -437,7 +440,7 @@ fn execute_sparse_struct( .cloned() .zip_eq(fill_values) .map(|(patch_values, fill_value)| unsafe { - SparseData::new_unchecked( + Sparse::new_unchecked( unresolved_patches .clone() .map_values(|_| Ok(patch_values)) @@ -474,8 +477,10 @@ fn execute_sparse_decimal( } } let filled_array = builder.finish_into_decimal(); - let array = filled_array.into_data().patch(patches, ctx)?; - Ok(array.into_array()) + let dtype = filled_array.dtype().clone(); + let len = filled_array.len(); + let data = filled_array.into_data().patch(patches, ctx)?; + Ok(Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data))?.into_array()) } fn execute_varbin( diff --git a/encodings/sparse/src/compute/cast.rs b/encodings/sparse/src/compute/cast.rs index 9806974979a..a9fe0046646 100644 --- a/encodings/sparse/src/compute/cast.rs +++ b/encodings/sparse/src/compute/cast.rs @@ -11,7 +11,6 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::Sparse; -use crate::SparseData; impl CastReduce for Sparse { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { let casted_patches = array @@ -28,7 +27,7 @@ impl CastReduce for Sparse { }; Ok(Some( - SparseData::try_new_from_patches(casted_patches, casted_fill)?.into_array(), + Sparse::try_new_from_patches(casted_patches, casted_fill)?.into_array(), )) } } diff --git a/encodings/sparse/src/compute/filter.rs b/encodings/sparse/src/compute/filter.rs index a3d3d5a1946..2cc8064a6cd 100644 --- a/encodings/sparse/src/compute/filter.rs +++ b/encodings/sparse/src/compute/filter.rs @@ -11,7 +11,6 @@ use vortex_mask::Mask; use crate::ConstantArray; use crate::Sparse; -use crate::SparseData; impl FilterKernel for Sparse { fn filter( array: ArrayView<'_, Self>, @@ -27,8 +26,7 @@ impl FilterKernel for Sparse { }; Ok(Some( - SparseData::try_new_from_patches(new_patches, array.fill_scalar().clone())? - .into_array(), + Sparse::try_new_from_patches(new_patches, array.fill_scalar().clone())?.into_array(), )) } } diff --git a/encodings/sparse/src/compute/take.rs b/encodings/sparse/src/compute/take.rs index ebcc2570fc1..b0b5f83cad0 100644 --- a/encodings/sparse/src/compute/take.rs +++ b/encodings/sparse/src/compute/take.rs @@ -10,7 +10,6 @@ use vortex_error::VortexResult; use crate::ConstantArray; use crate::Sparse; -use crate::SparseData; impl TakeExecute for Sparse { fn take( array: ArrayView<'_, Self>, @@ -40,7 +39,7 @@ impl TakeExecute for Sparse { } Ok(Some( - SparseData::try_new_from_patches( + Sparse::try_new_from_patches( new_patches, array.fill_scalar().cast( &array diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index 2d79238230b..fc9613a369d 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -10,6 +10,7 @@ use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; @@ -28,7 +29,6 @@ use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; use vortex_array::scalar_fn::fns::operators::Operator; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; @@ -57,15 +57,9 @@ mod slice; vtable!(Sparse, Sparse, SparseData); -#[derive(Debug)] -pub struct SparseMetadata { - patches: PatchesMetadata, - fill_value: Scalar, -} - #[derive(Clone, prost::Message)] #[repr(C)] -pub struct ProstPatchesMetadata { +pub struct SparseMetadata { #[prost(message, required, tag = "1")] patches: PatchesMetadata, } @@ -73,28 +67,15 @@ pub struct ProstPatchesMetadata { impl VTable for Sparse { type ArrayData = SparseData; - type Metadata = SparseMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Sparse - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &SparseData) -> usize { - array.patches.array_len() - } - - fn dtype(array: &SparseData) -> &DType { - array.fill_scalar().dtype() - } - - fn stats(array: &SparseData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + SparseData::validate(data.patches(), data.fill_scalar(), dtype, len) } fn array_hash(array: &SparseData, state: &mut H, precision: Precision) { @@ -128,32 +109,24 @@ impl VTable for Sparse { } } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { let patches = array.patches().to_metadata(array.len(), array.dtype())?; - - Ok(SparseMetadata { - patches, - fill_value: array.fill_value.clone(), - }) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - let prost_patches = ProstPatchesMetadata { - patches: metadata.patches, - }; + let metadata = SparseMetadata { patches }; // Note that we DO NOT serialize the fill value since that is stored in the buffers. - Ok(Some(prost_patches.encode_to_vec())) + Ok(Some(metadata.encode_to_vec())) } fn deserialize( - bytes: &[u8], + &self, dtype: &DType, - _len: usize, + len: usize, + metadata: &[u8], buffers: &[BufferHandle], + children: &dyn ArrayChildren, session: &VortexSession, - ) -> VortexResult { - let prost_patches = ProstPatchesMetadata::decode(bytes)?; + ) -> VortexResult { + let metadata = SparseMetadata::decode(metadata)?; // Once we have the patches metadata, we need to get the fill value from the buffers. @@ -165,19 +138,6 @@ impl VTable for Sparse { let scalar_value = ScalarValue::from_proto_bytes(scalar_bytes, dtype, session)?; let fill_value = Scalar::try_new(dtype.clone(), scalar_value)?; - Ok(SparseMetadata { - patches: prost_patches.patches, - fill_value, - }) - } - - fn build( - dtype: &DType, - len: usize, - metadata: &Self::Metadata, - _buffers: &[BufferHandle], - children: &dyn ArrayChildren, - ) -> VortexResult { vortex_ensure_eq!( children.len(), 2, @@ -200,7 +160,7 @@ impl VTable for Sparse { patch_values, None, )?, - metadata.fill_value.clone(), + fill_value, ) } @@ -276,7 +236,6 @@ pub struct SparseData { pub(crate) slots: Vec>, patches: Patches, fill_value: Scalar, - stats_set: ArrayStats, } #[derive(Clone, Debug)] @@ -292,7 +251,23 @@ impl Sparse { len: usize, fill_value: Scalar, ) -> VortexResult { - Array::try_from_data(SparseData::try_new(indices, values, len, fill_value)?) + let dtype = fill_value.dtype().clone(); + let data = SparseData::try_new(indices, values, len, fill_value)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Sparse, dtype, len, data)) }) + } + + pub fn try_new_from_patches(patches: Patches, fill_value: Scalar) -> VortexResult { + let dtype = fill_value.dtype().clone(); + let len = patches.array_len(); + let data = SparseData::try_new_from_patches(patches, fill_value)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Sparse, dtype, len, data)) }) + } + + pub(crate) unsafe fn new_unchecked(patches: Patches, fill_value: Scalar) -> SparseArray { + let dtype = fill_value.dtype().clone(); + let len = patches.array_len(); + let data = unsafe { SparseData::new_unchecked(patches, fill_value) }; + unsafe { Array::from_parts_unchecked(ArrayParts::new(Sparse, dtype, len, data)) } } /// Encode the given array as a [`SparseArray`]. @@ -302,6 +277,71 @@ impl Sparse { } impl SparseData { + fn normalize_values_dtype(values: ArrayRef, fill_value: &Scalar) -> VortexResult { + let fill_dtype = fill_value.dtype(); + let values_dtype = values.dtype(); + + vortex_ensure!( + values_dtype.eq_ignore_nullability(fill_dtype), + "fill value, {:?}, should be instance of values dtype, {} but was {}.", + fill_value, + values_dtype, + fill_dtype, + ); + + if values_dtype == fill_dtype { + Ok(values) + } else { + values.cast(fill_dtype.clone()) + } + } + + fn normalize_patches_dtype(patches: Patches, fill_value: &Scalar) -> VortexResult { + let fill_dtype = fill_value.dtype(); + let values_dtype = patches.values().dtype(); + + vortex_ensure!( + values_dtype.eq_ignore_nullability(fill_dtype), + "fill value, {:?}, should be instance of values dtype, {} but was {}.", + fill_value, + values_dtype, + fill_dtype, + ); + + if values_dtype == fill_dtype { + Ok(patches) + } else { + patches.cast_values(fill_dtype) + } + } + + pub fn validate( + patches: &Patches, + fill_value: &Scalar, + dtype: &DType, + len: usize, + ) -> VortexResult<()> { + vortex_ensure!( + fill_value.dtype() == dtype, + "fill value dtype {} does not match array dtype {}", + fill_value.dtype(), + dtype, + ); + vortex_ensure!( + patches.array_len() == len, + "patches length {} does not match array length {}", + patches.array_len(), + len + ); + vortex_ensure!( + patches.values().dtype() == dtype, + "patch values dtype {} does not match array dtype {}", + patches.values().dtype(), + dtype, + ); + Ok(()) + } + fn make_slots(patches: &Patches) -> Vec> { vec![ Some(patches.indices().clone()), @@ -341,27 +381,16 @@ impl SparseData { } } + let values = Self::normalize_values_dtype(values, &fill_value)?; + // TODO(0ax1): handle chunk offsets let patches = Patches::new(len, 0, indices, values, None)?; - let slots = Self::make_slots(&patches); - - Ok(Self { - slots, - patches, - fill_value, - stats_set: Default::default(), - }) + Self::try_new_from_patches(patches, fill_value) } /// Build a new SparseArray from an existing set of patches. pub fn try_new_from_patches(patches: Patches, fill_value: Scalar) -> VortexResult { - vortex_ensure!( - fill_value.dtype() == patches.values().dtype(), - "fill value, {:?}, should be instance of values dtype, {} but was {}.", - fill_value, - patches.values().dtype(), - fill_value.dtype(), - ); + let patches = Self::normalize_patches_dtype(patches, &fill_value)?; let slots = Self::make_slots(&patches); @@ -369,7 +398,6 @@ impl SparseData { slots, patches, fill_value, - stats_set: Default::default(), }) } @@ -380,7 +408,6 @@ impl SparseData { slots, patches, fill_value, - stats_set: Default::default(), } } @@ -436,10 +463,10 @@ impl SparseData { /// Optionally provided fill value will be respected if the array is less than 90% null. pub fn encode(array: &ArrayRef, fill_value: Option) -> VortexResult { if let Some(fill_value) = fill_value.as_ref() - && array.dtype() != fill_value.dtype() + && !array.dtype().eq_ignore_nullability(fill_value.dtype()) { vortex_bail!( - "Array and fill value types must match. got {} and {}", + "Array and fill value types must have the same base type. got {} and {}", array.dtype(), fill_value.dtype() ) @@ -474,17 +501,17 @@ impl SparseData { } }; - return Ok(SparseData::try_new( + return Sparse::try_new( non_null_indices, non_null_values, array.len(), Scalar::null(array.dtype().clone()), - )? - .into_array()); + ) + .map(IntoArray::into_array); } let fill = if let Some(fill) = fill_value { - fill + fill.cast(array.dtype())? } else { // TODO(robert): Support other dtypes, only thing missing is getting most common value out of the array let (top_pvalue, _) = array @@ -521,8 +548,8 @@ impl SparseData { Mask::Values(values) => values.indices().iter().map(|v| *v as u64).collect(), }; - SparseData::try_new(indices.into_array(), non_top_values, array.len(), fill) - .map(|a| a.into_array()) + Sparse::try_new(indices.into_array(), non_top_values, array.len(), fill) + .map(IntoArray::into_array) } } @@ -544,7 +571,7 @@ impl ValidityVTable for Sparse { }; Ok(Validity::Array( - unsafe { SparseData::new_unchecked(patches, array.fill_value.is_valid().into()) } + unsafe { Sparse::new_unchecked(patches, array.fill_value.is_valid().into()) } .into_array(), )) } diff --git a/encodings/sparse/src/rules.rs b/encodings/sparse/src/rules.rs index 91b4edf80b0..c53e7e4c6a5 100644 --- a/encodings/sparse/src/rules.rs +++ b/encodings/sparse/src/rules.rs @@ -12,7 +12,6 @@ use vortex_array::scalar_fn::fns::not::NotReduceAdaptor; use vortex_error::VortexResult; use crate::Sparse; -use crate::SparseData; pub(crate) static RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&CastReduceAdaptor(Sparse)), @@ -24,7 +23,7 @@ impl NotReduce for Sparse { let inverted_fill = array.fill_scalar().as_bool().invert().into_scalar(); let inverted_patches = array.patches().clone().map_values(|values| values.not())?; Ok(Some( - SparseData::try_new_from_patches(inverted_patches, inverted_fill)?.into_array(), + Sparse::try_new_from_patches(inverted_patches, inverted_fill)?.into_array(), )) } } diff --git a/encodings/sparse/src/slice.rs b/encodings/sparse/src/slice.rs index 6d06da58029..7a36d58e209 100644 --- a/encodings/sparse/src/slice.rs +++ b/encodings/sparse/src/slice.rs @@ -12,7 +12,6 @@ use vortex_error::VortexResult; use crate::ConstantArray; use crate::Sparse; -use crate::SparseData; impl SliceKernel for Sparse { fn slice( @@ -36,8 +35,7 @@ impl SliceKernel for Sparse { // patches slice will ensure that dtype of patches is unchanged and the indices and // values match Ok(Some( - unsafe { SparseData::new_unchecked(new_patches, array.fill_scalar().clone()) } - .into_array(), + unsafe { Sparse::new_unchecked(new_patches, array.fill_scalar().clone()) }.into_array(), )) } } diff --git a/encodings/zigzag/public-api.lock b/encodings/zigzag/public-api.lock index ab49a0acca7..ec8ed62f664 100644 --- a/encodings/zigzag/public-api.lock +++ b/encodings/zigzag/public-api.lock @@ -20,8 +20,6 @@ impl vortex_array::array::vtable::VTable for vortex_zigzag::ZigZag pub type vortex_zigzag::ZigZag::ArrayData = vortex_zigzag::ZigZagData -pub type vortex_zigzag::ZigZag::Metadata = vortex_array::metadata::EmptyMetadata - pub type vortex_zigzag::ZigZag::OperationsVTable = vortex_zigzag::ZigZag pub type vortex_zigzag::ZigZag::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -34,11 +32,7 @@ pub fn vortex_zigzag::ZigZag::buffer(_array: vortex_array::array::view::ArrayVie pub fn vortex_zigzag::ZigZag::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_zigzag::ZigZag::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_zigzag::ZigZag::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_zigzag::ZigZag::dtype(array: &vortex_zigzag::ZigZagData) -> &vortex_array::dtype::DType +pub fn vortex_zigzag::ZigZag::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_zigzag::ZigZag::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -46,23 +40,17 @@ pub fn vortex_zigzag::ZigZag::execute_parent(array: vortex_array::array::view::A pub fn vortex_zigzag::ZigZag::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_zigzag::ZigZag::len(array: &vortex_zigzag::ZigZagData) -> usize - -pub fn vortex_zigzag::ZigZag::metadata(_array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_zigzag::ZigZag::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_zigzag::ZigZag::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_zigzag::ZigZag::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_zigzag::ZigZag::serialize(_array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_zigzag::ZigZag::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_zigzag::ZigZag::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_zigzag::ZigZag::stats(array: &vortex_zigzag::ZigZagData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_zigzag::ZigZag::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_zigzag::ZigZag::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_zigzag::ZigZag::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -98,8 +86,6 @@ pub struct vortex_zigzag::ZigZagData impl vortex_zigzag::ZigZagData -pub fn vortex_zigzag::ZigZagData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_zigzag::ZigZagData::encoded(&self) -> &vortex_array::array::erased::ArrayRef pub fn vortex_zigzag::ZigZagData::is_empty(&self) -> bool @@ -116,18 +102,10 @@ impl core::clone::Clone for vortex_zigzag::ZigZagData pub fn vortex_zigzag::ZigZagData::clone(&self) -> vortex_zigzag::ZigZagData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_zigzag::ZigZagData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_zigzag::ZigZagData pub fn vortex_zigzag::ZigZagData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_zigzag::ZigZagData - -pub fn vortex_zigzag::ZigZagData::into_array(self) -> vortex_array::array::erased::ArrayRef - pub fn vortex_zigzag::zigzag_decode(parray: vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_array::arrays::primitive::vtable::PrimitiveArray pub fn vortex_zigzag::zigzag_encode(parray: vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult diff --git a/encodings/zigzag/src/array.rs b/encodings/zigzag/src/array.rs index 0cb2102c8c6..896afd67fb3 100644 --- a/encodings/zigzag/src/array.rs +++ b/encodings/zigzag/src/array.rs @@ -5,9 +5,9 @@ use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; -use vortex_array::EmptyMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; @@ -18,7 +18,6 @@ use vortex_array::dtype::PType; use vortex_array::match_each_unsigned_integer_ptype; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; use vortex_array::vtable::VTable; @@ -42,28 +41,25 @@ vtable!(ZigZag, ZigZag, ZigZagData); impl VTable for ZigZag { type ArrayData = ZigZagData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; - fn vtable(_array: &Self::ArrayData) -> &Self { - &ZigZag - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ZigZagData) -> usize { - array.encoded().len() - } - - fn dtype(array: &ZigZagData) -> &DType { - &array.dtype - } - - fn stats(array: &ZigZagData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + let expected_dtype = ZigZagData::dtype_from_encoded_dtype(data.encoded().dtype())?; + vortex_ensure!( + dtype == &expected_dtype, + "expected dtype {expected_dtype}, got {dtype}" + ); + vortex_ensure!( + data.encoded().len() == len, + "expected len {len}, got {}", + data.encoded().len() + ); + Ok(()) } fn array_hash(array: &ZigZagData, state: &mut H, precision: Precision) { @@ -86,31 +82,25 @@ impl VTable for ZigZag { vortex_panic!("ZigZagArray buffer_name index {idx} out of bounds") } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) - } - - fn build( + &self, dtype: &DType, len: usize, - _metadata: &Self::Metadata, + metadata: &[u8], _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + if !metadata.is_empty() { + vortex_bail!( + "ZigZagArray expects empty metadata, got {} bytes", + metadata.len() + ); + } if children.len() != 1 { vortex_bail!("Expected 1 child, got {}", children.len()); } @@ -172,9 +162,7 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["encoded"]; #[derive(Clone, Debug)] pub struct ZigZagData { - dtype: DType, pub(super) slots: Vec>, - stats_set: ArrayStats, } #[derive(Clone, Debug)] @@ -185,11 +173,19 @@ impl ZigZag { /// Construct a new [`ZigZagArray`] from an encoded unsigned integer array. pub fn try_new(encoded: ArrayRef) -> VortexResult { - Array::try_from_data(ZigZagData::try_new(encoded)?) + let data = ZigZagData::try_new(encoded)?; + let dtype = ZigZagData::dtype_from_encoded_dtype(data.encoded().dtype())?; + let len = data.len(); + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(ZigZag, dtype, len, data)) }) } } impl ZigZagData { + fn dtype_from_encoded_dtype(encoded_dtype: &DType) -> VortexResult { + Ok(DType::from(PType::try_from(encoded_dtype)?.to_signed()) + .with_nullability(encoded_dtype.nullability())) + } + pub fn new(encoded: ArrayRef) -> Self { Self::try_new(encoded).vortex_expect("ZigZagArray new") } @@ -200,13 +196,10 @@ impl ZigZagData { vortex_bail!(MismatchedTypes: "unsigned int", encoded_dtype); } - let dtype = DType::from(PType::try_from(&encoded_dtype)?.to_signed()) - .with_nullability(encoded_dtype.nullability()); + Self::dtype_from_encoded_dtype(&encoded_dtype)?; Ok(Self { - dtype, slots: vec![Some(encoded)], - stats_set: Default::default(), }) } @@ -222,14 +215,10 @@ impl ZigZagData { self.encoded().is_empty() } - /// Returns the logical data type of the array. - #[inline] - pub fn dtype(&self) -> &DType { - &self.dtype - } - pub fn ptype(&self) -> PType { - self.dtype().as_ptype() + PType::try_from(self.encoded().dtype()) + .vortex_expect("ZigZagArray encoded dtype") + .to_signed() } pub fn encoded(&self) -> &ArrayRef { diff --git a/encodings/zigzag/src/compress.rs b/encodings/zigzag/src/compress.rs index 690cd52deac..b2d142c755c 100644 --- a/encodings/zigzag/src/compress.rs +++ b/encodings/zigzag/src/compress.rs @@ -12,8 +12,8 @@ use vortex_error::vortex_bail; use vortex_error::vortex_panic; use zigzag::ZigZag as ExternalZigZag; +use crate::ZigZag; use crate::ZigZagArray; -use crate::ZigZagData; pub fn zigzag_encode(parray: PrimitiveArray) -> VortexResult { let validity = parray.validity(); let encoded = match parray.ptype() { @@ -26,7 +26,7 @@ pub fn zigzag_encode(parray: PrimitiveArray) -> VortexResult { parray.ptype() ), }; - ZigZagArray::try_from_data(ZigZagData::try_new(encoded.into_array())?) + ZigZag::try_new(encoded.into_array()) } fn zigzag_encode_primitive( diff --git a/encodings/zigzag/src/compute/cast.rs b/encodings/zigzag/src/compute/cast.rs index c69d91f5347..138c6aae736 100644 --- a/encodings/zigzag/src/compute/cast.rs +++ b/encodings/zigzag/src/compute/cast.rs @@ -10,7 +10,6 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::ZigZag; -use crate::ZigZagData; impl CastReduce for ZigZag { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { if !dtype.is_signed_int() { @@ -20,7 +19,7 @@ impl CastReduce for ZigZag { let new_encoded_dtype = DType::Primitive(dtype.as_ptype().to_unsigned(), dtype.nullability()); let new_encoded = array.encoded().cast(new_encoded_dtype)?; - Ok(Some(ZigZagData::try_new(new_encoded)?.into_array())) + Ok(Some(ZigZag::try_new(new_encoded)?.into_array())) } } diff --git a/encodings/zigzag/src/compute/mod.rs b/encodings/zigzag/src/compute/mod.rs index 9967ba5bb95..11fe580e6bc 100644 --- a/encodings/zigzag/src/compute/mod.rs +++ b/encodings/zigzag/src/compute/mod.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use crate::ZigZagData; mod cast; use vortex_array::ArrayRef; @@ -22,7 +21,7 @@ use crate::ZigZag; impl FilterReduce for ZigZag { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { let encoded = array.encoded().filter(mask.clone())?; - Ok(Some(ZigZagData::try_new(encoded)?.into_array())) + Ok(Some(ZigZag::try_new(encoded)?.into_array())) } } @@ -33,7 +32,7 @@ impl TakeExecute for ZigZag { _ctx: &mut ExecutionCtx, ) -> VortexResult> { let encoded = array.encoded().take(indices.clone())?; - Ok(Some(ZigZagData::try_new(encoded)?.into_array())) + Ok(Some(ZigZag::try_new(encoded)?.into_array())) } } @@ -44,7 +43,7 @@ impl MaskReduce for ZigZag { EmptyOptions, [array.encoded().clone(), mask.clone()], )?; - Ok(Some(ZigZagData::try_new(masked_encoded)?.into_array())) + Ok(Some(ZigZag::try_new(masked_encoded)?.into_array())) } } diff --git a/encodings/zigzag/src/slice.rs b/encodings/zigzag/src/slice.rs index 599d365809f..e027cda04cb 100644 --- a/encodings/zigzag/src/slice.rs +++ b/encodings/zigzag/src/slice.rs @@ -10,12 +10,11 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::ZigZag; -use crate::ZigZagData; impl SliceReduce for ZigZag { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { Ok(Some( - ZigZagData::new(array.encoded().slice(range)?).into_array(), + ZigZag::try_new(array.encoded().slice(range)?)?.into_array(), )) } } diff --git a/encodings/zstd/benches/listview_rebuild.rs b/encodings/zstd/benches/listview_rebuild.rs index 0e91342c8ac..55690cdccae 100644 --- a/encodings/zstd/benches/listview_rebuild.rs +++ b/encodings/zstd/benches/listview_rebuild.rs @@ -10,13 +10,17 @@ use vortex_array::arrays::VarBinViewArray; use vortex_array::arrays::listview::ListViewRebuildMode; use vortex_array::validity::Validity; use vortex_buffer::Buffer; +use vortex_zstd::Zstd; use vortex_zstd::ZstdData; #[divan::bench(sample_size = 1000)] fn rebuild_naive(bencher: Bencher) { let dudes = VarBinViewArray::from_iter_str(["Washington", "Adams", "Jefferson", "Madison"]) .into_array(); - let dudes = ZstdData::from_array(dudes, 9, 1024).unwrap().into_array(); + let dtype = dudes.dtype().clone(); + let dudes = Zstd::try_new(dtype, ZstdData::from_array(dudes, 9, 1024).unwrap()) + .unwrap() + .into_array(); let offsets = std::iter::repeat_n(0u32, 1024) .collect::>() diff --git a/encodings/zstd/public-api.lock b/encodings/zstd/public-api.lock index 29554686839..dc7e1e63809 100644 --- a/encodings/zstd/public-api.lock +++ b/encodings/zstd/public-api.lock @@ -6,12 +6,16 @@ impl vortex_zstd::Zstd pub const vortex_zstd::Zstd::ID: vortex_array::array::ArrayId +pub fn vortex_zstd::Zstd::decompress(array: &vortex_zstd::ZstdArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult + pub fn vortex_zstd::Zstd::from_primitive(parray: &vortex_array::arrays::primitive::vtable::PrimitiveArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult pub fn vortex_zstd::Zstd::from_var_bin_view(vbv: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult pub fn vortex_zstd::Zstd::from_var_bin_view_without_dict(vbv: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult +pub fn vortex_zstd::Zstd::try_new(dtype: vortex_array::dtype::DType, data: vortex_zstd::ZstdData) -> vortex_error::VortexResult + impl core::clone::Clone for vortex_zstd::Zstd pub fn vortex_zstd::Zstd::clone(&self) -> vortex_zstd::Zstd @@ -24,8 +28,6 @@ impl vortex_array::array::vtable::VTable for vortex_zstd::Zstd pub type vortex_zstd::Zstd::ArrayData = vortex_zstd::ZstdData -pub type vortex_zstd::Zstd::Metadata = vortex_array::metadata::ProstMetadata - pub type vortex_zstd::Zstd::OperationsVTable = vortex_zstd::Zstd pub type vortex_zstd::Zstd::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromValiditySliceHelper @@ -38,33 +40,23 @@ pub fn vortex_zstd::Zstd::buffer(array: vortex_array::array::view::ArrayView<'_, pub fn vortex_zstd::Zstd::buffer_name(array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_zstd::Zstd::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - -pub fn vortex_zstd::Zstd::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_zstd::Zstd::dtype(array: &vortex_zstd::ZstdData) -> &vortex_array::dtype::DType +pub fn vortex_zstd::Zstd::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_zstd::Zstd::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_zstd::Zstd::id(&self) -> vortex_array::array::ArrayId -pub fn vortex_zstd::Zstd::len(array: &vortex_zstd::ZstdData) -> usize - -pub fn vortex_zstd::Zstd::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult - pub fn vortex_zstd::Zstd::nbuffers(array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_zstd::Zstd::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_zstd::Zstd::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_zstd::Zstd::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_zstd::Zstd::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_zstd::Zstd::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_zstd::Zstd::stats(array: &vortex_zstd::ZstdData) -> &vortex_array::stats::array::ArrayStats - -pub fn vortex_zstd::Zstd::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_zstd::Zstd::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_zstd::Zstd::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -80,28 +72,6 @@ impl vortex_array::scalar_fn::fns::cast::kernel::CastReduce for vortex_zstd::Zst pub fn vortex_zstd::Zstd::cast(array: vortex_array::array::view::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> -pub struct vortex_zstd::ZstdArrayParts - -pub vortex_zstd::ZstdArrayParts::dictionary: core::option::Option - -pub vortex_zstd::ZstdArrayParts::dtype: vortex_array::dtype::DType - -pub vortex_zstd::ZstdArrayParts::frames: alloc::vec::Vec - -pub vortex_zstd::ZstdArrayParts::metadata: vortex_zstd::ZstdMetadata - -pub vortex_zstd::ZstdArrayParts::n_rows: usize - -pub vortex_zstd::ZstdArrayParts::slice_start: usize - -pub vortex_zstd::ZstdArrayParts::slice_stop: usize - -pub vortex_zstd::ZstdArrayParts::validity: vortex_array::validity::Validity - -impl core::fmt::Debug for vortex_zstd::ZstdArrayParts - -pub fn vortex_zstd::ZstdArrayParts::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub struct vortex_zstd::ZstdBuffersMetadata pub vortex_zstd::ZstdBuffersMetadata::buffer_alignments: alloc::vec::Vec @@ -134,10 +104,6 @@ pub struct vortex_zstd::ZstdData impl vortex_zstd::ZstdData -pub fn vortex_zstd::ZstdData::decompress(&self, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult - -pub fn vortex_zstd::ZstdData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_zstd::ZstdData::from_array(array: vortex_array::array::erased::ArrayRef, level: i32, values_per_frame: usize) -> vortex_error::VortexResult pub fn vortex_zstd::ZstdData::from_canonical(canonical: &vortex_array::canonical::Canonical, level: i32, values_per_frame: usize) -> vortex_error::VortexResult> @@ -150,34 +116,44 @@ pub fn vortex_zstd::ZstdData::from_var_bin_view(vbv: &vortex_array::arrays::varb pub fn vortex_zstd::ZstdData::from_var_bin_view_without_dict(vbv: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult -pub fn vortex_zstd::ZstdData::into_parts(self) -> vortex_zstd::ZstdArrayParts +pub fn vortex_zstd::ZstdData::into_parts(self) -> vortex_zstd::ZstdDataParts pub fn vortex_zstd::ZstdData::is_empty(&self) -> bool pub fn vortex_zstd::ZstdData::len(&self) -> usize -pub fn vortex_zstd::ZstdData::new(dictionary: core::option::Option, frames: alloc::vec::Vec, dtype: vortex_array::dtype::DType, metadata: vortex_zstd::ZstdMetadata, n_rows: usize, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_zstd::ZstdData::new(dictionary: core::option::Option, frames: alloc::vec::Vec, metadata: vortex_zstd::ZstdMetadata, n_rows: usize, validity: vortex_array::validity::Validity) -> Self + +pub fn vortex_zstd::ZstdData::validate(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> impl core::clone::Clone for vortex_zstd::ZstdData pub fn vortex_zstd::ZstdData::clone(&self) -> vortex_zstd::ZstdData -impl core::convert::From for vortex_array::array::erased::ArrayRef - -pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_zstd::ZstdData) -> vortex_array::array::erased::ArrayRef - impl core::fmt::Debug for vortex_zstd::ZstdData pub fn vortex_zstd::ZstdData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::array::IntoArray for vortex_zstd::ZstdData - -pub fn vortex_zstd::ZstdData::into_array(self) -> vortex_array::array::erased::ArrayRef - impl vortex_array::array::vtable::validity::ValiditySliceHelper for vortex_zstd::ZstdData pub fn vortex_zstd::ZstdData::unsliced_validity_and_slice(&self) -> (&vortex_array::validity::Validity, usize, usize) +pub struct vortex_zstd::ZstdDataParts + +pub vortex_zstd::ZstdDataParts::dictionary: core::option::Option + +pub vortex_zstd::ZstdDataParts::frames: alloc::vec::Vec + +pub vortex_zstd::ZstdDataParts::metadata: vortex_zstd::ZstdMetadata + +pub vortex_zstd::ZstdDataParts::n_rows: usize + +pub vortex_zstd::ZstdDataParts::slice_start: usize + +pub vortex_zstd::ZstdDataParts::slice_stop: usize + +pub vortex_zstd::ZstdDataParts::validity: vortex_array::validity::Validity + pub struct vortex_zstd::ZstdFrameMetadata pub vortex_zstd::ZstdFrameMetadata::n_values: u64 diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index d8c6875fa39..a494cfc572f 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -11,6 +11,7 @@ use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::Canonical; @@ -19,7 +20,6 @@ use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; use vortex_array::Precision; -use vortex_array::ProstMetadata; use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; @@ -32,7 +32,6 @@ use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; -use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; @@ -85,28 +84,15 @@ vtable!(Zstd, Zstd, ZstdData); impl VTable for Zstd { type ArrayData = ZstdData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromValiditySliceHelper; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Zstd - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ZstdData) -> usize { - array.slice_stop - array.slice_start - } - - fn dtype(array: &ZstdData) -> &DType { - &array.dtype - } - - fn stats(array: &ZstdData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + data.validate(dtype, len) } fn array_hash(array: &ZstdData, state: &mut H, precision: Precision) { @@ -179,31 +165,20 @@ impl VTable for Zstd { } } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(array.metadata.clone())) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.0.encode_to_vec())) + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some(array.metadata.clone().encode_to_vec())) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata(ZstdMetadata::decode(bytes)?)) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = ZstdMetadata::decode(metadata)?; let validity = if children.is_empty() { Validity::from(dtype.nullability()) } else if children.len() == 1 { @@ -213,7 +188,7 @@ impl VTable for Zstd { vortex_bail!("ZstdArray expected 0 or 1 child, got {}", children.len()); }; - let (dictionary_buffer, compressed_buffers) = if metadata.0.dictionary_size == 0 { + let (dictionary_buffer, compressed_buffers) = if metadata.dictionary_size == 0 { // no dictionary ( None, @@ -236,8 +211,7 @@ impl VTable for Zstd { Ok(ZstdData::new( dictionary_buffer, compressed_buffers, - dtype.clone(), - metadata.0.clone(), + metadata, len, validity, )) @@ -261,7 +235,7 @@ impl VTable for Zstd { array.unsliced_validity = match &slots[VALIDITY_SLOT] { Some(arr) => Validity::Array(arr.clone()), - None => Validity::from(array.dtype.nullability()), + None => Validity::from(array.unsliced_validity.nullability()), }; array.slots = slots; @@ -269,8 +243,7 @@ impl VTable for Zstd { } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { - array - .decompress(ctx)? + Zstd::decompress(&array, ctx)? .execute::(ctx) .map(ExecutionResult::done) } @@ -290,17 +263,22 @@ pub struct Zstd; impl Zstd { pub const ID: ArrayId = ArrayId::new_ref("vortex.zstd"); + pub fn try_new(dtype: DType, data: ZstdData) -> VortexResult { + let len = data.len(); + data.validate(&dtype, len)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Zstd, dtype, len, data)) }) + } + /// Compress a [`VarBinViewArray`] using Zstd without a dictionary. pub fn from_var_bin_view_without_dict( vbv: &VarBinViewArray, level: i32, values_per_frame: usize, ) -> VortexResult { - Array::try_from_data(ZstdData::from_var_bin_view_without_dict( - vbv, - level, - values_per_frame, - )?) + Self::try_new( + vbv.dtype().clone(), + ZstdData::from_var_bin_view_without_dict(vbv, level, values_per_frame)?, + ) } /// Compress a [`PrimitiveArray`] using Zstd. @@ -309,7 +287,10 @@ impl Zstd { level: i32, values_per_frame: usize, ) -> VortexResult { - Array::try_from_data(ZstdData::from_primitive(parray, level, values_per_frame)?) + Self::try_new( + parray.dtype().clone(), + ZstdData::from_primitive(parray, level, values_per_frame)?, + ) } /// Compress a [`VarBinViewArray`] using Zstd. @@ -318,7 +299,14 @@ impl Zstd { level: i32, values_per_frame: usize, ) -> VortexResult { - Array::try_from_data(ZstdData::from_var_bin_view(vbv, level, values_per_frame)?) + Self::try_new( + vbv.dtype().clone(), + ZstdData::from_var_bin_view(vbv, level, values_per_frame)?, + ) + } + + pub fn decompress(array: &ZstdArray, ctx: &mut ExecutionCtx) -> VortexResult { + array.data().decompress(array.dtype(), ctx) } } @@ -332,36 +320,25 @@ pub struct ZstdData { pub(crate) dictionary: Option, pub(crate) frames: Vec, pub(crate) metadata: ZstdMetadata, - dtype: DType, pub(crate) unsliced_validity: Validity, unsliced_n_rows: usize, pub(super) slots: Vec>, - stats_set: ArrayStats, slice_start: usize, slice_stop: usize, } -/// The parts of a [`ZstdArray`] returned by [`ZstdArray::into_parts`]. -#[derive(Debug)] -pub struct ZstdArrayParts { - /// The optional dictionary used for compression. +pub struct ZstdDataParts { pub dictionary: Option, - /// The compressed frames. pub frames: Vec, - /// The compression metadata. pub metadata: ZstdMetadata, - /// The data type of the uncompressed array. - pub dtype: DType, - /// The validity of the uncompressed array. pub validity: Validity, - /// The number of rows in the uncompressed array. pub n_rows: usize, - /// Slice start offset. pub slice_start: usize, - /// Slice stop offset. pub slice_stop: usize, } +/// The parts of a [`ZstdArray`] returned by [`ZstdArray::into_parts`]. +#[derive(Debug)] struct Frames { dictionary: Option, frames: Vec, @@ -462,7 +439,6 @@ impl ZstdData { pub fn new( dictionary: Option, frames: Vec, - dtype: DType, metadata: ZstdMetadata, n_rows: usize, validity: Validity, @@ -472,16 +448,94 @@ impl ZstdData { dictionary, frames, metadata, - dtype, unsliced_validity: validity, unsliced_n_rows: n_rows, slots: vec![validity_slot], - stats_set: Default::default(), slice_start: 0, slice_stop: n_rows, } } + pub fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + matches!( + dtype, + DType::Primitive(..) | DType::Binary(_) | DType::Utf8(_) + ), + "Unsupported dtype for Zstd array: {dtype}" + ); + vortex_ensure!( + self.slice_start <= self.slice_stop, + "Invalid slice range {}..{}", + self.slice_start, + self.slice_stop + ); + vortex_ensure!( + self.slice_stop <= self.unsliced_n_rows, + "Slice stop {} exceeds unsliced row count {}", + self.slice_stop, + self.unsliced_n_rows + ); + vortex_ensure!( + self.slice_stop - self.slice_start == len, + "Slice length {} does not match array length {}", + self.slice_stop - self.slice_start, + len + ); + if let Some(validity_len) = self.unsliced_validity.maybe_len() { + vortex_ensure!( + validity_len == self.unsliced_n_rows, + "Validity length {} does not match unsliced row count {}", + validity_len, + self.unsliced_n_rows + ); + } + + match &self.dictionary { + Some(dictionary) => vortex_ensure!( + usize::try_from(self.metadata.dictionary_size)? == dictionary.len(), + "Dictionary size metadata {} does not match buffer size {}", + self.metadata.dictionary_size, + dictionary.len() + ), + None => vortex_ensure!( + self.metadata.dictionary_size == 0, + "Dictionary metadata present without dictionary buffer" + ), + } + vortex_ensure!( + self.frames.len() == self.metadata.frames.len(), + "Frame count {} does not match metadata frame count {}", + self.frames.len(), + self.metadata.frames.len() + ); + + Ok(()) + } + + pub(crate) fn with_slice(&self, start: usize, stop: usize) -> Self { + let new_start = self.slice_start + start; + let new_stop = self.slice_start + stop; + + assert!( + new_start <= self.slice_stop, + "new slice start {new_start} exceeds end {}", + self.slice_stop + ); + + assert!( + new_stop <= self.slice_stop, + "new slice stop {new_stop} exceeds end {}", + self.slice_stop + ); + + Self { + slice_start: new_start, + slice_stop: new_stop, + ..self.clone() + } + } + fn compress_values( value_bytes: &ByteBuffer, frame_byte_starts: &[usize], @@ -585,7 +639,6 @@ impl ZstdData { values_per_frame: usize, use_dictionary: bool, ) -> VortexResult { - let dtype = parray.dtype().clone(); let byte_width = parray.ptype().byte_width(); // We compress only the valid elements. @@ -629,7 +682,6 @@ impl ZstdData { Ok(ZstdData::new( dictionary, frames, - dtype, metadata, parray.len(), parray.validity(), @@ -682,8 +734,6 @@ impl ZstdData { // to separate the binary data and lengths as two separate streams, but // this approach is simpler and can be best in cases when there is // mutual information between strings and their lengths. - let dtype = vbv.dtype().clone(); - // We compress only the valid elements. let (value_bytes, value_byte_indices) = collect_valid_vbv(vbv)?; let n_values = value_byte_indices.len(); @@ -720,7 +770,6 @@ impl ZstdData { Ok(ZstdData::new( dictionary, frames, - dtype, metadata, vbv.len(), vbv.validity(), @@ -752,18 +801,18 @@ impl ZstdData { .ok_or_else(|| vortex_err!("Zstd can only encode Primitive and VarBinView arrays")) } - fn byte_width(&self) -> usize { - if self.dtype.is_primitive() { - self.dtype.as_ptype().byte_width() + fn byte_width(dtype: &DType) -> usize { + if dtype.is_primitive() { + dtype.as_ptype().byte_width() } else { 1 } } - pub fn decompress(&self, ctx: &mut ExecutionCtx) -> VortexResult { + fn decompress(&self, dtype: &DType, ctx: &mut ExecutionCtx) -> VortexResult { // To start, we figure out which frames we need to decompress, and with // what row offset into the first such frame. - let byte_width = self.byte_width(); + let byte_width = Self::byte_width(dtype); let slice_n_rows = self.slice_stop - self.slice_start; let slice_value_indices = self .unsliced_validity @@ -845,21 +894,21 @@ impl ZstdData { // // We ensure that the validity of the decompressed array ALWAYS matches the validity // implied by the DType. - if !self.dtype().is_nullable() && !matches!(slice_validity, Validity::NonNullable) { + if !dtype.is_nullable() && !matches!(slice_validity, Validity::NonNullable) { assert!( matches!(slice_validity, Validity::AllValid), "ZSTD array expects to be non-nullable but there are nulls after decompression" ); slice_validity = Validity::NonNullable; - } else if self.dtype.is_nullable() && matches!(slice_validity, Validity::NonNullable) { + } else if dtype.is_nullable() && matches!(slice_validity, Validity::NonNullable) { slice_validity = Validity::AllValid; } // // END OF IMPORTANT BLOCK // - match &self.dtype { + match dtype { DType::Primitive(..) => { let slice_values_buffer = decompressed.slice( (slice_value_idx_start - n_skipped_values) * byte_width @@ -867,7 +916,7 @@ impl ZstdData { ); let primitive = PrimitiveArray::from_values_byte_buffer( slice_values_buffer, - self.dtype.as_ptype(), + dtype.as_ptype(), slice_validity, slice_n_rows, ); @@ -888,14 +937,14 @@ impl ZstdData { VarBinViewArray::new_unchecked( valid_views, Arc::from(buffers), - self.dtype.clone(), + dtype.clone(), slice_validity, ) } .into_array()) } AllOr::None => Ok(ConstantArray::new( - Scalar::null(self.dtype.clone()), + Scalar::null(dtype.clone()), slice_n_rows, ) .into_array()), @@ -916,7 +965,7 @@ impl ZstdData { VarBinViewArray::new_unchecked( views.freeze(), Arc::from(buffers), - self.dtype.clone(), + dtype.clone(), slice_validity, ) } @@ -924,46 +973,7 @@ impl ZstdData { } } } - _ => vortex_panic!("Unsupported dtype for Zstd array: {}", self.dtype), - } - } - - pub(crate) fn _slice(&self, start: usize, stop: usize) -> ZstdArray { - let new_start = self.slice_start + start; - let new_stop = self.slice_start + stop; - - assert!( - new_start <= self.slice_stop, - "new slice start {new_start} exceeds end {}", - self.slice_stop - ); - - assert!( - new_stop <= self.slice_stop, - "new slice stop {new_stop} exceeds end {}", - self.slice_stop - ); - - Array::try_from_data(ZstdData { - slice_start: self.slice_start + start, - slice_stop: self.slice_start + stop, - stats_set: Default::default(), - ..self.clone() - }) - .vortex_expect("ZstdData is always valid") - } - - /// Consumes the array and returns its parts. - pub fn into_parts(self) -> ZstdArrayParts { - ZstdArrayParts { - dictionary: self.dictionary, - frames: self.frames, - metadata: self.metadata, - dtype: self.dtype, - validity: self.unsliced_validity, - n_rows: self.unsliced_n_rows, - slice_start: self.slice_start, - slice_stop: self.slice_stop, + _ => vortex_panic!("Unsupported dtype for Zstd array: {}", dtype), } } @@ -979,10 +989,16 @@ impl ZstdData { self.slice_stop == self.slice_start } - /// Returns the logical data type of the array. - #[inline] - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn into_parts(self) -> ZstdDataParts { + ZstdDataParts { + dictionary: self.dictionary, + frames: self.frames, + metadata: self.metadata, + validity: self.unsliced_validity, + n_rows: self.unsliced_n_rows, + slice_start: self.slice_start, + slice_stop: self.slice_stop, + } } pub(crate) fn slice_start(&self) -> usize { @@ -1011,10 +1027,11 @@ impl OperationsVTable for Zstd { _ctx: &mut ExecutionCtx, ) -> VortexResult { let mut ctx = LEGACY_SESSION.create_execution_ctx(); - array - ._slice(index, index + 1) - .decompress(&mut ctx)? - .scalar_at(0) + let sliced = Zstd::try_new( + array.dtype().clone(), + array.data().with_slice(index, index + 1), + )?; + Zstd::decompress(&sliced, &mut ctx)?.scalar_at(0) } } diff --git a/encodings/zstd/src/compute/cast.rs b/encodings/zstd/src/compute/cast.rs index feadd131e95..4ad636fe88a 100644 --- a/encodings/zstd/src/compute/cast.rs +++ b/encodings/zstd/src/compute/cast.rs @@ -34,14 +34,16 @@ impl CastReduce for Zstd { (Nullability::NonNullable, Nullability::Nullable) => { // nonnull => null, trivial cast by altering the validity Ok(Some( - ZstdData::new( - array.dictionary.clone(), - array.frames.clone(), + Zstd::try_new( dtype.clone(), - array.metadata.clone(), - array.unsliced_n_rows(), - array.unsliced_validity.clone(), - ) + ZstdData::new( + array.dictionary.clone(), + array.frames.clone(), + array.metadata.clone(), + array.unsliced_n_rows(), + array.unsliced_validity.clone(), + ), + )? .into_array() .slice(array.slice_start()..array.slice_stop())?, )) @@ -60,14 +62,16 @@ impl CastReduce for Zstd { // If there are no nulls, the cast is trivial Ok(Some( - ZstdData::new( - array.dictionary.clone(), - array.frames.clone(), + Zstd::try_new( dtype.clone(), - array.metadata.clone(), - array.unsliced_n_rows(), - array.unsliced_validity.clone(), - ) + ZstdData::new( + array.dictionary.clone(), + array.frames.clone(), + array.metadata.clone(), + array.unsliced_n_rows(), + array.unsliced_validity.clone(), + ), + )? .into_array() .slice(array.slice_start()..array.slice_stop())?, )) diff --git a/encodings/zstd/src/slice.rs b/encodings/zstd/src/slice.rs index fad72ae3279..2f74d40072c 100644 --- a/encodings/zstd/src/slice.rs +++ b/encodings/zstd/src/slice.rs @@ -13,6 +13,12 @@ use crate::Zstd; impl SliceReduce for Zstd { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - Ok(Some(array._slice(range.start, range.end).into_array())) + Ok(Some( + Zstd::try_new( + array.dtype().clone(), + array.data().with_slice(range.start, range.end), + )? + .into_array(), + )) } } diff --git a/encodings/zstd/src/test.rs b/encodings/zstd/src/test.rs index e95ce025667..7548f32cb80 100644 --- a/encodings/zstd/src/test.rs +++ b/encodings/zstd/src/test.rs @@ -32,7 +32,7 @@ fn test_zstd_compress_decompress() { // check full decompression works let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let decompressed = compressed.decompress(&mut ctx).unwrap(); + let decompressed = Zstd::decompress(&compressed, &mut ctx).unwrap(); assert_arrays_eq!(decompressed, PrimitiveArray::from_iter(data)); // check slicing works @@ -78,7 +78,9 @@ fn test_zstd_with_validity_and_multi_frame() { assert_nth_scalar!(compressed, 177, 177); let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let decompressed = compressed.decompress(&mut ctx).unwrap().to_primitive(); + let decompressed = Zstd::decompress(&compressed, &mut ctx) + .unwrap() + .to_primitive(); let decompressed_values = decompressed.as_slice::(); assert_eq!(decompressed_values[3], 3); assert_eq!(decompressed_values[177], 177); @@ -121,7 +123,9 @@ fn test_zstd_with_dict() { assert_nth_scalar!(compressed, 199, 199); let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let decompressed = compressed.decompress(&mut ctx).unwrap().to_primitive(); + let decompressed = Zstd::decompress(&compressed, &mut ctx) + .unwrap() + .to_primitive(); assert_arrays_eq!(decompressed, PrimitiveArray::from_iter(data)); // check slicing works @@ -193,7 +197,9 @@ fn test_zstd_decompress_var_bin_view() { assert_nth_scalar!(compressed, 4, "baz"); let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let decompressed = compressed.decompress(&mut ctx).unwrap().to_varbinview(); + let decompressed = Zstd::decompress(&compressed, &mut ctx) + .unwrap() + .to_varbinview(); assert_nth_scalar!(decompressed, 0, "foo"); assert_nth_scalar!(decompressed, 1, "bar"); assert_nth_scalar!(decompressed, 2, None::); diff --git a/encodings/zstd/src/zstd_buffers.rs b/encodings/zstd/src/zstd_buffers.rs index 4f5b8a9b918..1e74a69a9a0 100644 --- a/encodings/zstd/src/zstd_buffers.rs +++ b/encodings/zstd/src/zstd_buffers.rs @@ -10,19 +10,17 @@ use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; +use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; -use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::ProstMetadata; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; use vortex_array::session::ArraySessionExt; -use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; use vortex_array::vtable::VTable; @@ -44,6 +42,67 @@ pub struct ZstdBuffers; impl ZstdBuffers { pub const ID: ArrayId = ArrayId::new_ref("vortex.zstd_buffers"); + + pub fn try_new( + dtype: DType, + len: usize, + data: ZstdBuffersData, + ) -> VortexResult { + Array::try_from_parts(ArrayParts::new(ZstdBuffers, dtype, len, data)) + } + + pub fn compress(array: &ArrayRef, level: i32) -> VortexResult { + let encoding_id = array.encoding_id(); + let metadata = array + .metadata()? + .ok_or_else(|| vortex_err!("Array does not support serialization"))?; + let buffer_handles = array.buffer_handles(); + let children = array.children(); + + let mut compressed_buffers = Vec::with_capacity(buffer_handles.len()); + let mut uncompressed_sizes = Vec::with_capacity(buffer_handles.len()); + let mut buffer_alignments = Vec::with_capacity(buffer_handles.len()); + + let mut compressor = zstd::bulk::Compressor::new(level)?; + // Compression is currently CPU-only, so we gather all buffers on the host. + for handle in &buffer_handles { + buffer_alignments.push(u32::from(handle.alignment())); + let host_buf = handle.clone().try_to_host_sync()?; + uncompressed_sizes.push(host_buf.len() as u64); + let compressed = compressor.compress(&host_buf)?; + compressed_buffers.push(BufferHandle::new_host(ByteBuffer::from(compressed))); + } + + let data = ZstdBuffersData { + inner_encoding_id: encoding_id, + inner_metadata: metadata, + compressed_buffers, + uncompressed_sizes, + buffer_alignments, + slots: children.into_iter().map(Some).collect(), + }; + let compressed = Self::try_new(array.dtype().clone(), array.len(), data)?; + compressed.statistics().inherit_from(array.statistics()); + Ok(compressed) + } + + pub fn build_inner( + array: &ZstdBuffersArray, + buffer_handles: &[BufferHandle], + session: &VortexSession, + ) -> VortexResult { + array + .data() + .build_inner(array.dtype(), array.len(), buffer_handles, session) + } + + fn decompress_and_build_inner( + array: &ZstdBuffersArray, + session: &VortexSession, + ) -> VortexResult { + let decompressed_buffers = array.data().decompress_buffers()?; + Self::build_inner(array, &decompressed_buffers, session) + } } /// An encoding that ZSTD-compresses the buffers of any wrapped array. @@ -59,9 +118,6 @@ pub struct ZstdBuffersData { uncompressed_sizes: Vec, buffer_alignments: Vec, pub(crate) slots: Vec>, - dtype: DType, - len: usize, - stats_set: ArrayStats, } #[derive(Clone, Debug)] @@ -124,24 +180,6 @@ impl ZstdBuffersDecodePlan { } impl ZstdBuffersData { - /// Returns the length of the array. - #[inline] - pub fn len(&self) -> usize { - self.len - } - - /// Returns whether the array is empty. - #[inline] - pub fn is_empty(&self) -> bool { - self.len == 0 - } - - /// Returns the logical data type of the array. - #[inline] - pub fn dtype(&self) -> &DType { - &self.dtype - } - fn validate(&self) -> VortexResult<()> { vortex_ensure_eq!( self.compressed_buffers.len(), @@ -160,51 +198,6 @@ impl ZstdBuffersData { Ok(()) } - /// Compresses the buffers of the given array using ZSTD. - /// - /// Each buffer of the input array is independently ZSTD-compressed. The children - /// and metadata of the input array are preserved as-is. - pub fn compress(array: &ArrayRef, level: i32) -> VortexResult { - let encoding_id = array.encoding_id(); - let metadata = array - .metadata()? - .ok_or_else(|| vortex_err!("Array does not support serialization"))?; - let buffer_handles = array.buffer_handles(); - let children = array.children(); - - let mut compressed_buffers = Vec::with_capacity(buffer_handles.len()); - let mut uncompressed_sizes = Vec::with_capacity(buffer_handles.len()); - let mut buffer_alignments = Vec::with_capacity(buffer_handles.len()); - - let mut compressor = zstd::bulk::Compressor::new(level)?; - // Compression is currently CPU-only, so we gather all buffers on the host. - for handle in &buffer_handles { - buffer_alignments.push(u32::from(handle.alignment())); - let host_buf = handle.clone().try_to_host_sync()?; - uncompressed_sizes.push(host_buf.len() as u64); - let compressed = compressor.compress(&host_buf)?; - compressed_buffers.push(BufferHandle::new_host(ByteBuffer::from(compressed))); - } - - let compressed = Self { - inner_encoding_id: encoding_id, - inner_metadata: metadata, - compressed_buffers, - uncompressed_sizes, - buffer_alignments, - slots: children.into_iter().map(Some).collect(), - dtype: array.dtype().clone(), - len: array.len(), - stats_set: Default::default(), - }; - let compressed_ref = compressed.clone().into_array(); - compressed - .stats_set - .to_ref(&compressed_ref) - .inherit_from(array.statistics()); - Ok(compressed) - } - fn decompress_buffers(&self) -> VortexResult> { // CPU decode path: zstd::bulk works on host bytes, so compressed buffers are // materialized on the host via `try_to_host_sync`. @@ -252,15 +245,22 @@ impl ZstdBuffersData { Ok(result) } - fn decompress_and_build_inner(&self, session: &VortexSession) -> VortexResult { + fn decompress_and_build_inner( + &self, + dtype: &DType, + len: usize, + session: &VortexSession, + ) -> VortexResult { let decompressed_buffers = self.decompress_buffers()?; - self.build_inner(&decompressed_buffers, session) + self.build_inner(dtype, len, &decompressed_buffers, session) } // This is exposed to help non-CPU executors pass uncompressed buffer handles // to build the inner array. pub fn build_inner( &self, + dtype: &DType, + len: usize, buffer_handles: &[BufferHandle], session: &VortexSession, ) -> VortexResult { @@ -272,8 +272,8 @@ impl ZstdBuffersData { let children: Vec = self.slots.iter().flatten().cloned().collect(); inner_vtable.build( self.inner_encoding_id.clone(), - &self.dtype, - self.len, + dtype, + len, &self.inner_metadata, buffer_handles, &children.as_slice(), @@ -346,29 +346,15 @@ fn array_id_from_string(s: &str) -> ArrayId { impl VTable for ZstdBuffers { type ArrayData = ZstdBuffersData; - - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &ZstdBuffers - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ZstdBuffersData) -> usize { - array.len - } - - fn dtype(array: &ZstdBuffersData) -> &DType { - &array.dtype - } - - fn stats(array: &ZstdBuffersData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, _dtype: &DType, _len: usize) -> VortexResult<()> { + data.validate() } fn array_hash( @@ -433,36 +419,28 @@ impl VTable for ZstdBuffers { Ok(()) } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(ZstdBuffersMetadata { - inner_encoding_id: array.inner_encoding_id.to_string(), - inner_metadata: array.inner_metadata.clone(), - uncompressed_sizes: array.uncompressed_sizes.clone(), - buffer_alignments: array.buffer_alignments.clone(), - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.0.encode_to_vec())) + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some( + ZstdBuffersMetadata { + inner_encoding_id: array.inner_encoding_id.to_string(), + inner_metadata: array.inner_metadata.clone(), + uncompressed_sizes: array.uncompressed_sizes.clone(), + buffer_alignments: array.buffer_alignments.clone(), + } + .encode_to_vec(), + )) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata(ZstdBuffersMetadata::decode(bytes)?)) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = ZstdBuffersMetadata::decode(metadata)?; let compressed_buffers: Vec = buffers.to_vec(); let child_arrays: Vec> = (0..children.len()) @@ -470,15 +448,12 @@ impl VTable for ZstdBuffers { .collect::>>()?; let data = ZstdBuffersData { - inner_encoding_id: array_id_from_string(&metadata.0.inner_encoding_id), - inner_metadata: metadata.0.inner_metadata.clone(), + inner_encoding_id: array_id_from_string(&metadata.inner_encoding_id), + inner_metadata: metadata.inner_metadata.clone(), compressed_buffers, - uncompressed_sizes: metadata.0.uncompressed_sizes.clone(), - buffer_alignments: metadata.0.buffer_alignments.clone(), + uncompressed_sizes: metadata.uncompressed_sizes.clone(), + buffer_alignments: metadata.buffer_alignments.clone(), slots: child_arrays, - dtype: dtype.clone(), - len, - stats_set: Default::default(), }; data.validate()?; @@ -489,7 +464,7 @@ impl VTable for ZstdBuffers { fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { let session = ctx.session(); - let inner_array = array.decompress_and_build_inner(session)?; + let inner_array = ZstdBuffers::decompress_and_build_inner(&array, session)?; inner_array .execute::(ctx) .map(ExecutionResult::done) @@ -505,7 +480,11 @@ impl OperationsVTable for ZstdBuffers { // TODO(os): maybe we should not support scalar_at, it is really slow, and adding a cache // layer here is weird. Valid use of zstd buffers array would be by executing it first into // canonical - let inner_array = array.decompress_and_build_inner(&vortex_array::LEGACY_SESSION)?; + let inner_array = array.data().decompress_and_build_inner( + array.dtype(), + array.len(), + &vortex_array::LEGACY_SESSION, + )?; inner_array.scalar_at(index) } } @@ -514,11 +493,15 @@ impl ValidityVTable for ZstdBuffers { fn validity( array: ArrayView<'_, ZstdBuffers>, ) -> VortexResult { - if !array.dtype.is_nullable() { + if !array.dtype().is_nullable() { return Ok(vortex_array::validity::Validity::NonNullable); } - let inner_array = array.decompress_and_build_inner(&vortex_array::LEGACY_SESSION)?; + let inner_array = array.data().decompress_and_build_inner( + array.dtype(), + array.len(), + &vortex_array::LEGACY_SESSION, + )?; inner_array.validity() } } @@ -580,10 +563,10 @@ mod tests { #[case::empty_primitive(make_empty_primitive_array())] #[case::inlined_varbinview(make_inlined_varbinview_array())] fn test_roundtrip(#[case] input: ArrayRef) -> VortexResult<()> { - let compressed = ZstdBuffersData::compress(&input, 3)?; + let compressed = ZstdBuffers::compress(&input, 3)?; - assert_eq!(compressed.len, input.len()); - assert_eq!(&compressed.dtype, input.dtype()); + assert_eq!(compressed.len(), input.len()); + assert_eq!(compressed.dtype(), input.dtype()); let mut ctx = LEGACY_SESSION.create_execution_ctx(); let decompressed = compressed.into_array().execute::(&mut ctx)?; @@ -597,8 +580,7 @@ mod tests { let input = make_primitive_array(); input.statistics().set(Stat::Min, Precision::exact(0i32)); - let compressed = ZstdBuffersData::compress(&input, 3)?; - let compressed = ZstdBuffersArray::try_from_data(compressed)?; + let compressed = ZstdBuffers::compress(&input, 3)?; assert!(compressed.statistics().get(Stat::Min).is_some()); Ok(()) @@ -607,7 +589,7 @@ mod tests { #[test] fn test_validity_delegates_for_nullable_input() -> VortexResult<()> { let input = make_nullable_primitive_array(); - let compressed = ZstdBuffersData::compress(&input, 3)?.into_array(); + let compressed = ZstdBuffers::compress(&input, 3)?.into_array(); assert_eq!(compressed.all_valid()?, input.all_valid()?); assert_eq!(compressed.all_invalid()?, input.all_invalid()?); diff --git a/fuzz/src/array/filter.rs b/fuzz/src/array/filter.rs index 7db3e1f8f74..9431cac68f6 100644 --- a/fuzz/src/array/filter.rs +++ b/fuzz/src/array/filter.rs @@ -98,7 +98,7 @@ pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult StructArray::try_new_with_dtype( filtered_children, - struct_array.struct_fields().clone(), + struct_array.struct_fields(), filter.iter().filter(|b| **b).map(|b| *b as usize).sum(), validity, ) diff --git a/fuzz/src/array/mask.rs b/fuzz/src/array/mask.rs index 02974d72df7..89192b3e360 100644 --- a/fuzz/src/array/mask.rs +++ b/fuzz/src/array/mask.rs @@ -117,7 +117,7 @@ pub fn mask_canonical_array(canonical: Canonical, mask: &Mask) -> VortexResult>>()?; StructArray::try_new_with_dtype( sliced_children, - struct_array.struct_fields().clone(), + struct_array.struct_fields(), stop - start, validity, ) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 5189514d877..7e6487c34d1 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -864,8 +864,6 @@ impl vortex_array::VTable for vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ArrayData = vortex_array::arrays::bool::BoolData -pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Bool::OperationsVTable = vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool @@ -880,25 +878,19 @@ pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Bool::dtype(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Bool::len(array: &vortex_array::arrays::bool::BoolData) -> usize +pub fn vortex_array::arrays::Bool::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Bool::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -908,15 +900,13 @@ pub fn vortex_array::arrays::Bool::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Bool::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -954,25 +944,15 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::Bo pub fn vortex_array::arrays::Bool::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::Bool>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::bool::BoolArrayParts - -pub vortex_array::arrays::bool::BoolArrayParts::bits: vortex_array::buffer::BufferHandle - -pub vortex_array::arrays::bool::BoolArrayParts::len: usize - -pub vortex_array::arrays::bool::BoolArrayParts::offset: usize - -pub vortex_array::arrays::bool::BoolArrayParts::validity: vortex_array::validity::Validity - pub struct vortex_array::arrays::bool::BoolData impl vortex_array::arrays::bool::BoolData -pub fn vortex_array::arrays::bool::BoolData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::bool::BoolData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::bool::BoolData::into_bit_buffer(self) -> vortex_buffer::bit::buf::BitBuffer -pub fn vortex_array::arrays::bool::BoolData::into_parts(self) -> vortex_array::arrays::bool::BoolArrayParts +pub fn vortex_array::arrays::bool::BoolData::into_parts(self) -> vortex_array::arrays::bool::BoolDataParts pub fn vortex_array::arrays::bool::BoolData::is_empty(&self) -> bool @@ -994,17 +974,19 @@ impl core::clone::Clone for vortex_array::arrays::bool::BoolData pub fn vortex_array::arrays::bool::BoolData::clone(&self) -> vortex_array::arrays::bool::BoolData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::bool::BoolData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::bool::BoolData pub fn vortex_array::arrays::bool::BoolData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::bool::BoolData +pub struct vortex_array::arrays::bool::BoolDataParts + +pub vortex_array::arrays::bool::BoolDataParts::bits: vortex_array::buffer::BufferHandle + +pub vortex_array::arrays::bool::BoolDataParts::len: usize -pub fn vortex_array::arrays::bool::BoolData::into_array(self) -> vortex_array::ArrayRef +pub vortex_array::arrays::bool::BoolDataParts::offset: usize + +pub vortex_array::arrays::bool::BoolDataParts::validity: vortex_array::validity::Validity pub struct vortex_array::arrays::bool::BoolMaskedValidityRule @@ -1048,8 +1030,6 @@ impl vortex_array::VTable for vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ArrayData = vortex_array::arrays::chunked::ChunkedData -pub type vortex_array::arrays::Chunked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Chunked::OperationsVTable = vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::Chunked @@ -1064,25 +1044,19 @@ pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Chunked::dtype(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Chunked::len(array: &vortex_array::arrays::chunked::ChunkedData) -> usize +pub fn vortex_array::arrays::Chunked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Chunked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1092,15 +1066,13 @@ pub fn vortex_array::arrays::Chunked::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Chunked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -1174,18 +1146,10 @@ impl core::clone::Clone for vortex_array::arrays::chunked::ChunkedData pub fn vortex_array::arrays::chunked::ChunkedData::clone(&self) -> vortex_array::arrays::chunked::ChunkedData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::chunked::ChunkedData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::chunked::ChunkedData pub fn vortex_array::arrays::chunked::ChunkedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::chunked::ChunkedData - -pub fn vortex_array::arrays::chunked::ChunkedData::into_array(self) -> vortex_array::ArrayRef - pub type vortex_array::arrays::chunked::ChunkedArray = vortex_array::Array pub mod vortex_array::arrays::constant @@ -1216,8 +1180,6 @@ impl vortex_array::VTable for vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ArrayData = vortex_array::arrays::constant::ConstantData -pub type vortex_array::arrays::Constant::Metadata = vortex_array::scalar::Scalar - pub type vortex_array::arrays::Constant::OperationsVTable = vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays::Constant @@ -1232,25 +1194,19 @@ pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Constant::dtype(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Constant::len(array: &vortex_array::arrays::constant::ConstantData) -> usize +pub fn vortex_array::arrays::Constant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Constant::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1260,15 +1216,13 @@ pub fn vortex_array::arrays::Constant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Constant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -1308,15 +1262,9 @@ pub struct vortex_array::arrays::constant::ConstantData impl vortex_array::arrays::constant::ConstantData -pub fn vortex_array::arrays::constant::ConstantData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_array::arrays::constant::ConstantData::into_parts(self) -> vortex_array::scalar::Scalar -pub fn vortex_array::arrays::constant::ConstantData::is_empty(&self) -> bool - -pub fn vortex_array::arrays::constant::ConstantData::len(&self) -> usize - -pub fn vortex_array::arrays::constant::ConstantData::new(scalar: S, len: usize) -> Self where S: core::convert::Into +pub fn vortex_array::arrays::constant::ConstantData::new(scalar: S) -> Self where S: core::convert::Into pub fn vortex_array::arrays::constant::ConstantData::scalar(&self) -> &vortex_array::scalar::Scalar @@ -1324,18 +1272,10 @@ impl core::clone::Clone for vortex_array::arrays::constant::ConstantData pub fn vortex_array::arrays::constant::ConstantData::clone(&self) -> vortex_array::arrays::constant::ConstantData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::constant::ConstantData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::constant::ConstantData pub fn vortex_array::arrays::constant::ConstantData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::constant::ConstantData - -pub fn vortex_array::arrays::constant::ConstantData::into_array(self) -> vortex_array::ArrayRef - pub type vortex_array::arrays::constant::ConstantArray = vortex_array::Array pub mod vortex_array::arrays::datetime @@ -1422,8 +1362,6 @@ impl vortex_array::VTable for vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ArrayData = vortex_array::arrays::decimal::DecimalData -pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Decimal::OperationsVTable = vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::Decimal @@ -1438,25 +1376,19 @@ pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Decimal::dtype(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Decimal::len(array: &vortex_array::arrays::decimal::DecimalData) -> usize +pub fn vortex_array::arrays::Decimal::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Decimal::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1466,15 +1398,13 @@ pub fn vortex_array::arrays::Decimal::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -1512,16 +1442,6 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::De pub fn vortex_array::arrays::Decimal::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::decimal::DecimalArrayParts - -pub vortex_array::arrays::decimal::DecimalArrayParts::decimal_dtype: vortex_array::dtype::DecimalDType - -pub vortex_array::arrays::decimal::DecimalArrayParts::validity: vortex_array::validity::Validity - -pub vortex_array::arrays::decimal::DecimalArrayParts::values: vortex_array::buffer::BufferHandle - -pub vortex_array::arrays::decimal::DecimalArrayParts::values_type: vortex_array::dtype::DecimalType - pub struct vortex_array::arrays::decimal::DecimalData impl vortex_array::arrays::decimal::DecimalData @@ -1532,13 +1452,13 @@ pub fn vortex_array::arrays::decimal::DecimalData::buffer_handle(&self) -> &vort pub fn vortex_array::arrays::decimal::DecimalData::decimal_dtype(&self) -> vortex_array::dtype::DecimalDType -pub fn vortex_array::arrays::decimal::DecimalData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::decimal::DecimalData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::decimal::DecimalData::from_iter>(iter: I, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self pub fn vortex_array::arrays::decimal::DecimalData::from_option_iter>>(iter: I, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self -pub fn vortex_array::arrays::decimal::DecimalData::into_parts(self) -> vortex_array::arrays::decimal::DecimalArrayParts +pub fn vortex_array::arrays::decimal::DecimalData::into_parts(self) -> vortex_array::arrays::decimal::DecimalDataParts pub fn vortex_array::arrays::decimal::DecimalData::is_empty(&self) -> bool @@ -1572,17 +1492,19 @@ impl core::clone::Clone for vortex_array::arrays::decimal::DecimalData pub fn vortex_array::arrays::decimal::DecimalData::clone(&self) -> vortex_array::arrays::decimal::DecimalData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::decimal::DecimalData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::decimal::DecimalData pub fn vortex_array::arrays::decimal::DecimalData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::decimal::DecimalData +pub struct vortex_array::arrays::decimal::DecimalDataParts -pub fn vortex_array::arrays::decimal::DecimalData::into_array(self) -> vortex_array::ArrayRef +pub vortex_array::arrays::decimal::DecimalDataParts::decimal_dtype: vortex_array::dtype::DecimalDType + +pub vortex_array::arrays::decimal::DecimalDataParts::validity: vortex_array::validity::Validity + +pub vortex_array::arrays::decimal::DecimalDataParts::values: vortex_array::buffer::BufferHandle + +pub vortex_array::arrays::decimal::DecimalDataParts::values_type: vortex_array::dtype::DecimalType pub struct vortex_array::arrays::decimal::DecimalMaskedValidityRule @@ -1630,8 +1552,6 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData -pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -1646,25 +1566,19 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize +pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1674,15 +1588,13 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -1746,8 +1658,6 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData -pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -1762,25 +1672,19 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize +pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1790,15 +1694,13 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -1838,25 +1740,17 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::di pub fn vortex_array::arrays::dict::Dict::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::dict::Dict>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::dict::DictArrayParts - -pub vortex_array::arrays::dict::DictArrayParts::codes: vortex_array::ArrayRef - -pub vortex_array::arrays::dict::DictArrayParts::dtype: vortex_array::dtype::DType - -pub vortex_array::arrays::dict::DictArrayParts::values: vortex_array::ArrayRef - pub struct vortex_array::arrays::dict::DictData impl vortex_array::arrays::dict::DictData pub fn vortex_array::arrays::dict::DictData::codes(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::dict::DictData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::dict::DictData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::dict::DictData::has_all_values_referenced(&self) -> bool -pub fn vortex_array::arrays::dict::DictData::into_parts(self) -> vortex_array::arrays::dict::DictArrayParts +pub fn vortex_array::arrays::dict::DictData::into_parts(self) -> vortex_array::arrays::dict::DictDataParts pub fn vortex_array::arrays::dict::DictData::is_empty(&self) -> bool @@ -1868,8 +1762,6 @@ pub unsafe fn vortex_array::arrays::dict::DictData::new_unchecked(codes: vortex_ pub unsafe fn vortex_array::arrays::dict::DictData::set_all_values_referenced(self, all_values_referenced: bool) -> Self -pub fn vortex_array::arrays::dict::DictData::try_new(codes: vortex_array::ArrayRef, values: vortex_array::ArrayRef) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::DictData::validate_all_values_referenced(&self) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::dict::DictData::values(&self) -> &vortex_array::ArrayRef @@ -1882,17 +1774,15 @@ impl core::clone::Clone for vortex_array::arrays::dict::DictData pub fn vortex_array::arrays::dict::DictData::clone(&self) -> vortex_array::arrays::dict::DictData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::dict::DictData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::dict::DictData pub fn vortex_array::arrays::dict::DictData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::dict::DictData +pub struct vortex_array::arrays::dict::DictDataParts -pub fn vortex_array::arrays::dict::DictData::into_array(self) -> vortex_array::ArrayRef +pub vortex_array::arrays::dict::DictDataParts::codes: vortex_array::ArrayRef + +pub vortex_array::arrays::dict::DictDataParts::values: vortex_array::ArrayRef pub struct vortex_array::arrays::dict::DictMetadata @@ -2056,8 +1946,6 @@ impl vortex_array::VTable for vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData -pub type vortex_array::arrays::Extension::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -2072,25 +1960,19 @@ pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Extension::dtype(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Extension::len(array: &vortex_array::arrays::extension::ExtensionData) -> usize +pub fn vortex_array::arrays::Extension::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Extension::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2100,15 +1982,13 @@ pub fn vortex_array::arrays::Extension::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Extension::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Extension::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -2144,7 +2024,7 @@ pub struct vortex_array::arrays::extension::ExtensionData impl vortex_array::arrays::extension::ExtensionData -pub fn vortex_array::arrays::extension::ExtensionData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::extension::ExtensionData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::extension::ExtensionData::ext_dtype(&self) -> &vortex_array::dtype::extension::ExtDTypeRef @@ -2164,18 +2044,10 @@ impl core::clone::Clone for vortex_array::arrays::extension::ExtensionData pub fn vortex_array::arrays::extension::ExtensionData::clone(&self) -> vortex_array::arrays::extension::ExtensionData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::extension::ExtensionData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::extension::ExtensionData pub fn vortex_array::arrays::extension::ExtensionData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::into_array(self) -> vortex_array::ArrayRef - pub type vortex_array::arrays::extension::ExtensionArray = vortex_array::Array pub mod vortex_array::arrays::filter @@ -2202,8 +2074,6 @@ impl vortex_array::VTable for vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ArrayData = vortex_array::arrays::filter::FilterData -pub type vortex_array::arrays::Filter::Metadata = vortex_array::arrays::filter::vtable::FilterMetadata - pub type vortex_array::arrays::Filter::OperationsVTable = vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Filter @@ -2218,25 +2088,19 @@ pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::filter::vtable::FilterMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Filter::dtype(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Filter::len(array: &vortex_array::arrays::filter::FilterData) -> usize +pub fn vortex_array::arrays::Filter::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Filter::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2246,15 +2110,13 @@ pub fn vortex_array::arrays::Filter::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Filter::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Filter::vtable(_array: &vortex_array::arrays::filter::FilterData) -> &Self +pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -2262,12 +2124,6 @@ impl vortex_array::ValidityVTable for vortex_array pub fn vortex_array::arrays::Filter::validity(array: vortex_array::ArrayView<'_, vortex_array::arrays::Filter>) -> vortex_error::VortexResult -pub struct vortex_array::arrays::filter::FilterArrayParts - -pub vortex_array::arrays::filter::FilterArrayParts::child: vortex_array::ArrayRef - -pub vortex_array::arrays::filter::FilterArrayParts::mask: vortex_mask::Mask - pub struct vortex_array::arrays::filter::FilterData impl vortex_array::arrays::filter::FilterData @@ -2278,33 +2134,27 @@ pub fn vortex_array::arrays::filter::FilterData::dtype(&self) -> &vortex_array:: pub fn vortex_array::arrays::filter::FilterData::filter_mask(&self) -> &vortex_mask::Mask +pub fn vortex_array::arrays::filter::FilterData::into_parts(self) -> vortex_array::arrays::filter::FilterDataParts + pub fn vortex_array::arrays::filter::FilterData::is_empty(&self) -> bool pub fn vortex_array::arrays::filter::FilterData::len(&self) -> usize pub fn vortex_array::arrays::filter::FilterData::new(array: vortex_array::ArrayRef, mask: vortex_mask::Mask) -> Self -pub fn vortex_array::arrays::filter::FilterData::try_new(array: vortex_array::ArrayRef, mask: vortex_mask::Mask) -> vortex_error::VortexResult - -impl vortex_array::arrays::filter::FilterData - -pub fn vortex_array::arrays::filter::FilterData::into_parts(self) -> vortex_array::arrays::filter::FilterArrayParts - impl core::clone::Clone for vortex_array::arrays::filter::FilterData pub fn vortex_array::arrays::filter::FilterData::clone(&self) -> vortex_array::arrays::filter::FilterData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::filter::FilterData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::filter::FilterData pub fn vortex_array::arrays::filter::FilterData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::filter::FilterData +pub struct vortex_array::arrays::filter::FilterDataParts + +pub vortex_array::arrays::filter::FilterDataParts::child: vortex_array::ArrayRef -pub fn vortex_array::arrays::filter::FilterData::into_array(self) -> vortex_array::ArrayRef +pub vortex_array::arrays::filter::FilterDataParts::mask: vortex_mask::Mask pub struct vortex_array::arrays::filter::FilterExecuteAdaptor(pub V) @@ -2412,8 +2262,6 @@ impl vortex_array::VTable for vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ArrayData = vortex_array::arrays::fixed_size_list::FixedSizeListData -pub type vortex_array::arrays::FixedSizeList::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::FixedSizeList::OperationsVTable = vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arrays::FixedSizeList @@ -2428,25 +2276,19 @@ pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::FixedSizeList::dtype(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::FixedSizeList::len(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> usize +pub fn vortex_array::arrays::FixedSizeList::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::FixedSizeList::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2456,15 +2298,13 @@ pub fn vortex_array::arrays::FixedSizeList::reduce(array: vortex_array::ArrayVie pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::FixedSizeList::vtable(_array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &Self +pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -2492,7 +2332,7 @@ pub struct vortex_array::arrays::fixed_size_list::FixedSizeListData impl vortex_array::arrays::fixed_size_list::FixedSizeListData -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::elements(&self) -> &vortex_array::ArrayRef @@ -2508,8 +2348,6 @@ pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::new(elements: v pub unsafe fn vortex_array::arrays::fixed_size_list::FixedSizeListData::new_unchecked(elements: vortex_array::ArrayRef, list_size: u32, validity: vortex_array::validity::Validity, len: usize) -> Self -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::try_new(elements: vortex_array::ArrayRef, list_size: u32, validity: vortex_array::validity::Validity, len: usize) -> vortex_error::VortexResult - pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::validate(elements: &vortex_array::ArrayRef, len: usize, list_size: u32, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::validity(&self) -> vortex_array::validity::Validity @@ -2524,18 +2362,10 @@ impl core::clone::Clone for vortex_array::arrays::fixed_size_list::FixedSizeList pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::clone(&self) -> vortex_array::arrays::fixed_size_list::FixedSizeListData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::fixed_size_list::FixedSizeListData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::fixed_size_list::FixedSizeListData pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::fixed_size_list::FixedSizeListData - -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::into_array(self) -> vortex_array::ArrayRef - pub type vortex_array::arrays::fixed_size_list::FixedSizeListArray = vortex_array::Array pub mod vortex_array::arrays::list @@ -2562,8 +2392,6 @@ impl vortex_array::VTable for vortex_array::arrays::List pub type vortex_array::arrays::List::ArrayData = vortex_array::arrays::list::ListData -pub type vortex_array::arrays::List::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::List::OperationsVTable = vortex_array::arrays::List pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List @@ -2578,25 +2406,19 @@ pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::List::dtype(array: &vortex_array::arrays::list::ListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::List::len(array: &vortex_array::arrays::list::ListData) -> usize +pub fn vortex_array::arrays::List::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::List::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2606,15 +2428,13 @@ pub fn vortex_array::arrays::List::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::List::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::list::ListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::List::vtable(_array: &vortex_array::arrays::list::ListData) -> &Self +pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -2642,27 +2462,17 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::Li pub fn vortex_array::arrays::List::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::list::ListArrayParts - -pub vortex_array::arrays::list::ListArrayParts::dtype: vortex_array::dtype::DType - -pub vortex_array::arrays::list::ListArrayParts::elements: vortex_array::ArrayRef - -pub vortex_array::arrays::list::ListArrayParts::offsets: vortex_array::ArrayRef - -pub vortex_array::arrays::list::ListArrayParts::validity: vortex_array::validity::Validity - pub struct vortex_array::arrays::list::ListData impl vortex_array::arrays::list::ListData -pub fn vortex_array::arrays::list::ListData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::list::ListData::dtype(&self) -> vortex_array::dtype::DType -pub fn vortex_array::arrays::list::ListData::element_dtype(&self) -> &alloc::sync::Arc +pub fn vortex_array::arrays::list::ListData::element_dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::list::ListData::elements(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::list::ListData::into_parts(self) -> vortex_array::arrays::list::ListArrayParts +pub fn vortex_array::arrays::list::ListData::into_parts(self) -> vortex_array::arrays::list::ListDataParts pub fn vortex_array::arrays::list::ListData::is_empty(&self) -> bool @@ -2680,8 +2490,6 @@ pub fn vortex_array::arrays::list::ListData::offsets(&self) -> &vortex_array::Ar pub fn vortex_array::arrays::list::ListData::sliced_elements(&self) -> vortex_error::VortexResult -pub fn vortex_array::arrays::list::ListData::try_new(elements: vortex_array::ArrayRef, offsets: vortex_array::ArrayRef, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult - pub fn vortex_array::arrays::list::ListData::validate(elements: &vortex_array::ArrayRef, offsets: &vortex_array::ArrayRef, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::list::ListData::validity(&self) -> vortex_array::validity::Validity @@ -2696,17 +2504,19 @@ impl core::clone::Clone for vortex_array::arrays::list::ListData pub fn vortex_array::arrays::list::ListData::clone(&self) -> vortex_array::arrays::list::ListData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::list::ListData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::list::ListData pub fn vortex_array::arrays::list::ListData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::list::ListData +pub struct vortex_array::arrays::list::ListDataParts + +pub vortex_array::arrays::list::ListDataParts::dtype: vortex_array::dtype::DType + +pub vortex_array::arrays::list::ListDataParts::elements: vortex_array::ArrayRef + +pub vortex_array::arrays::list::ListDataParts::offsets: vortex_array::ArrayRef -pub fn vortex_array::arrays::list::ListData::into_array(self) -> vortex_array::ArrayRef +pub vortex_array::arrays::list::ListDataParts::validity: vortex_array::validity::Validity pub type vortex_array::arrays::list::ListArray = vortex_array::Array @@ -2744,8 +2554,6 @@ impl vortex_array::VTable for vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ArrayData = vortex_array::arrays::listview::ListViewData -pub type vortex_array::arrays::ListView::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::ListView::OperationsVTable = vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays::ListView @@ -2760,25 +2568,19 @@ pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::ListView::dtype(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::ListView::len(array: &vortex_array::arrays::listview::ListViewData) -> usize +pub fn vortex_array::arrays::ListView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::ListView::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2788,15 +2590,13 @@ pub fn vortex_array::arrays::ListView::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::ListView::vtable(_array: &vortex_array::arrays::listview::ListViewData) -> &Self +pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -2820,27 +2620,15 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::Li pub fn vortex_array::arrays::ListView::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::listview::ListViewArrayParts - -pub vortex_array::arrays::listview::ListViewArrayParts::elements: vortex_array::ArrayRef - -pub vortex_array::arrays::listview::ListViewArrayParts::elements_dtype: alloc::sync::Arc - -pub vortex_array::arrays::listview::ListViewArrayParts::offsets: vortex_array::ArrayRef - -pub vortex_array::arrays::listview::ListViewArrayParts::sizes: vortex_array::ArrayRef - -pub vortex_array::arrays::listview::ListViewArrayParts::validity: vortex_array::validity::Validity - pub struct vortex_array::arrays::listview::ListViewData impl vortex_array::arrays::listview::ListViewData -pub fn vortex_array::arrays::listview::ListViewData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::listview::ListViewData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::listview::ListViewData::elements(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::listview::ListViewData::into_parts(self) -> vortex_array::arrays::listview::ListViewArrayParts +pub fn vortex_array::arrays::listview::ListViewData::into_parts(self) -> vortex_array::arrays::listview::ListViewDataParts pub fn vortex_array::arrays::listview::ListViewData::is_empty(&self) -> bool @@ -2878,17 +2666,21 @@ impl core::clone::Clone for vortex_array::arrays::listview::ListViewData pub fn vortex_array::arrays::listview::ListViewData::clone(&self) -> vortex_array::arrays::listview::ListViewData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::listview::ListViewData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::listview::ListViewData pub fn vortex_array::arrays::listview::ListViewData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::listview::ListViewData +pub struct vortex_array::arrays::listview::ListViewDataParts + +pub vortex_array::arrays::listview::ListViewDataParts::elements: vortex_array::ArrayRef -pub fn vortex_array::arrays::listview::ListViewData::into_array(self) -> vortex_array::ArrayRef +pub vortex_array::arrays::listview::ListViewDataParts::elements_dtype: alloc::sync::Arc + +pub vortex_array::arrays::listview::ListViewDataParts::offsets: vortex_array::ArrayRef + +pub vortex_array::arrays::listview::ListViewDataParts::sizes: vortex_array::ArrayRef + +pub vortex_array::arrays::listview::ListViewDataParts::validity: vortex_array::validity::Validity pub fn vortex_array::arrays::listview::list_from_list_view(list_view: vortex_array::arrays::ListViewArray) -> vortex_error::VortexResult @@ -2922,8 +2714,6 @@ impl vortex_array::VTable for vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ArrayData = vortex_array::arrays::masked::MaskedData -pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked @@ -2938,25 +2728,19 @@ pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Masked::dtype(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Masked::len(array: &vortex_array::arrays::masked::MaskedData) -> usize +pub fn vortex_array::arrays::Masked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Masked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2966,15 +2750,13 @@ pub fn vortex_array::arrays::Masked::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Masked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Masked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3004,14 +2786,12 @@ impl vortex_array::arrays::masked::MaskedData pub fn vortex_array::arrays::masked::MaskedData::child(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::masked::MaskedData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::masked::MaskedData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::masked::MaskedData::is_empty(&self) -> bool pub fn vortex_array::arrays::masked::MaskedData::len(&self) -> usize -pub fn vortex_array::arrays::masked::MaskedData::try_new(child: vortex_array::ArrayRef, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult - pub fn vortex_array::arrays::masked::MaskedData::validity(&self) -> vortex_array::validity::Validity pub fn vortex_array::arrays::masked::MaskedData::validity_mask(&self) -> vortex_mask::Mask @@ -3020,18 +2800,10 @@ impl core::clone::Clone for vortex_array::arrays::masked::MaskedData pub fn vortex_array::arrays::masked::MaskedData::clone(&self) -> vortex_array::arrays::masked::MaskedData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::masked::MaskedData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::masked::MaskedData pub fn vortex_array::arrays::masked::MaskedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::masked::MaskedData - -pub fn vortex_array::arrays::masked::MaskedData::into_array(self) -> vortex_array::ArrayRef - pub fn vortex_array::arrays::masked::mask_validity_canonical(canonical: vortex_array::Canonical, validity_mask: &vortex_mask::Mask, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub type vortex_array::arrays::masked::MaskedArray = vortex_array::Array @@ -3064,8 +2836,6 @@ impl vortex_array::VTable for vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ArrayData = vortex_array::arrays::null::NullData -pub type vortex_array::arrays::null::Null::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::null::Null::OperationsVTable = vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays::null::Null @@ -3080,25 +2850,19 @@ pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::null::Null::dtype(_array: &vortex_array::arrays::null::NullData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::null::Null::len(array: &vortex_array::arrays::null::NullData) -> usize +pub fn vortex_array::arrays::null::Null::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::null::Null::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3108,15 +2872,13 @@ pub fn vortex_array::arrays::null::Null::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::null::Null::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::null::Null::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3148,30 +2910,20 @@ pub struct vortex_array::arrays::null::NullData impl vortex_array::arrays::null::NullData -pub fn vortex_array::arrays::null::NullData::dtype(&self) -> &vortex_array::dtype::DType - -pub fn vortex_array::arrays::null::NullData::is_empty(&self) -> bool - -pub fn vortex_array::arrays::null::NullData::len(&self) -> usize - -pub fn vortex_array::arrays::null::NullData::new(len: usize) -> Self +pub fn vortex_array::arrays::null::NullData::new() -> Self impl core::clone::Clone for vortex_array::arrays::null::NullData pub fn vortex_array::arrays::null::NullData::clone(&self) -> vortex_array::arrays::null::NullData -impl core::convert::From for vortex_array::ArrayRef +impl core::default::Default for vortex_array::arrays::null::NullData -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::null::NullData) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::null::NullData::default() -> Self impl core::fmt::Debug for vortex_array::arrays::null::NullData pub fn vortex_array::arrays::null::NullData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::null::NullData - -pub fn vortex_array::arrays::null::NullData::into_array(self) -> vortex_array::ArrayRef - pub type vortex_array::arrays::null::NullArray = vortex_array::Array pub mod vortex_array::arrays::patched @@ -3194,8 +2946,6 @@ impl vortex_array::VTable for vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray -pub type vortex_array::arrays::patched::Patched::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -3210,25 +2960,19 @@ pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::patched::Patched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::patched::Patched::len(array: &Self::ArrayData) -> usize +pub fn vortex_array::arrays::patched::Patched::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::patched::Patched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3238,15 +2982,13 @@ pub fn vortex_array::arrays::patched::Patched::reduce(array: vortex_array::Array pub fn vortex_array::arrays::patched::Patched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::patched::Patched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::patched::Patched::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3276,8 +3018,12 @@ impl vortex_array::arrays::patched::PatchedArray pub fn vortex_array::arrays::patched::PatchedArray::base_array(&self) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::PatchedArray::is_empty(&self) -> bool + pub fn vortex_array::arrays::patched::PatchedArray::lane_offsets(&self) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::PatchedArray::len(&self) -> usize + pub fn vortex_array::arrays::patched::PatchedArray::patch_indices(&self) -> &vortex_array::ArrayRef pub fn vortex_array::arrays::patched::PatchedArray::patch_values(&self) -> &vortex_array::ArrayRef @@ -3288,7 +3034,7 @@ pub fn vortex_array::arrays::patched::PatchedArray::from_array_and_patches(inner impl vortex_array::arrays::patched::PatchedArray -pub fn vortex_array::arrays::patched::PatchedArray::to_array(&self) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::PatchedArray::to_array(&self) -> vortex_array::ArrayRef where Self: core::clone::Clone + vortex_array::IntoArray impl core::clone::Clone for vortex_array::arrays::patched::PatchedArray @@ -3296,7 +3042,7 @@ pub fn vortex_array::arrays::patched::PatchedArray::clone(&self) -> vortex_array impl core::convert::From for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> vortex_array::ArrayRef +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> Self impl core::fmt::Debug for vortex_array::arrays::patched::PatchedArray @@ -3416,8 +3162,6 @@ impl vortex_array::VTable for vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ArrayData = vortex_array::arrays::primitive::PrimitiveData -pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive @@ -3432,25 +3176,19 @@ pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Primitive::dtype(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Primitive::len(array: &vortex_array::arrays::primitive::PrimitiveData) -> usize +pub fn vortex_array::arrays::Primitive::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Primitive::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3460,15 +3198,13 @@ pub fn vortex_array::arrays::Primitive::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3506,14 +3242,6 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::Pr pub fn vortex_array::arrays::Primitive::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::primitive::PrimitiveArrayParts - -pub vortex_array::arrays::primitive::PrimitiveArrayParts::buffer: vortex_array::buffer::BufferHandle - -pub vortex_array::arrays::primitive::PrimitiveArrayParts::ptype: vortex_array::dtype::PType - -pub vortex_array::arrays::primitive::PrimitiveArrayParts::validity: vortex_array::validity::Validity - pub struct vortex_array::arrays::primitive::PrimitiveData impl vortex_array::arrays::primitive::PrimitiveData @@ -3526,7 +3254,7 @@ impl vortex_array::arrays::primitive::PrimitiveData pub fn vortex_array::arrays::primitive::PrimitiveData::buffer_handle(&self) -> &vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::primitive::PrimitiveData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::primitive::PrimitiveData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::primitive::PrimitiveData::from_buffer_handle(handle: vortex_array::buffer::BufferHandle, ptype: vortex_array::dtype::PType, validity: vortex_array::validity::Validity) -> Self @@ -3572,23 +3300,23 @@ pub fn vortex_array::arrays::primitive::PrimitiveData::validate vortex_array::arrays::primitive::PrimitiveArrayParts +pub fn vortex_array::arrays::primitive::PrimitiveData::into_parts(self) -> vortex_array::arrays::primitive::PrimitiveDataParts impl core::clone::Clone for vortex_array::arrays::primitive::PrimitiveData pub fn vortex_array::arrays::primitive::PrimitiveData::clone(&self) -> vortex_array::arrays::primitive::PrimitiveData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::primitive::PrimitiveData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::primitive::PrimitiveData pub fn vortex_array::arrays::primitive::PrimitiveData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::primitive::PrimitiveData +pub struct vortex_array::arrays::primitive::PrimitiveDataParts + +pub vortex_array::arrays::primitive::PrimitiveDataParts::buffer: vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::primitive::PrimitiveData::into_array(self) -> vortex_array::ArrayRef +pub vortex_array::arrays::primitive::PrimitiveDataParts::ptype: vortex_array::dtype::PType + +pub vortex_array::arrays::primitive::PrimitiveDataParts::validity: vortex_array::validity::Validity pub struct vortex_array::arrays::primitive::PrimitiveMaskedValidityRule @@ -3664,16 +3392,10 @@ impl vortex_array::arrays::scalar_fn::ScalarFnData pub fn vortex_array::arrays::scalar_fn::ScalarFnData::children(&self) -> alloc::vec::Vec -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::dtype(&self) -> &vortex_array::dtype::DType - pub fn vortex_array::arrays::scalar_fn::ScalarFnData::get_child(&self, idx: usize) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::is_empty(&self) -> bool - pub fn vortex_array::arrays::scalar_fn::ScalarFnData::iter_children(&self) -> impl core::iter::traits::iterator::Iterator + '_ -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::len(&self) -> usize - pub fn vortex_array::arrays::scalar_fn::ScalarFnData::nchildren(&self) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnData::scalar_fn(&self) -> &vortex_array::scalar_fn::ScalarFnRef @@ -3684,18 +3406,10 @@ impl core::clone::Clone for vortex_array::arrays::scalar_fn::ScalarFnData pub fn vortex_array::arrays::scalar_fn::ScalarFnData::clone(&self) -> vortex_array::arrays::scalar_fn::ScalarFnData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::scalar_fn::ScalarFnData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::scalar_fn::ScalarFnData pub fn vortex_array::arrays::scalar_fn::ScalarFnData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::scalar_fn::ScalarFnData - -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::into_array(self) -> vortex_array::ArrayRef - pub struct vortex_array::arrays::scalar_fn::ScalarFnVTable impl core::clone::Clone for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -3714,8 +3428,6 @@ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::Metadata = vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata - pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -3730,25 +3442,19 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::dtype(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::len(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> usize +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3758,15 +3464,13 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_arr pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::vtable(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &Self +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3808,8 +3512,6 @@ impl vortex_array::VTable for vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ArrayData = vortex_array::arrays::shared::SharedData -pub type vortex_array::arrays::Shared::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Shared::OperationsVTable = vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Shared @@ -3824,25 +3526,19 @@ pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Shared::dtype(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Shared::len(array: &vortex_array::arrays::shared::SharedData) -> usize +pub fn vortex_array::arrays::Shared::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Shared::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3852,15 +3548,13 @@ pub fn vortex_array::arrays::Shared::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Shared::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Shared::vtable(_array: &vortex_array::arrays::shared::SharedData) -> &Self +pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3888,18 +3582,10 @@ impl core::clone::Clone for vortex_array::arrays::shared::SharedData pub fn vortex_array::arrays::shared::SharedData::clone(&self) -> vortex_array::arrays::shared::SharedData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::shared::SharedData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::shared::SharedData pub fn vortex_array::arrays::shared::SharedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::shared::SharedData - -pub fn vortex_array::arrays::shared::SharedData::into_array(self) -> vortex_array::ArrayRef - pub type vortex_array::arrays::shared::SharedArray = vortex_array::Array pub mod vortex_array::arrays::slice @@ -3926,8 +3612,6 @@ impl vortex_array::VTable for vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ArrayData = vortex_array::arrays::slice::SliceData -pub type vortex_array::arrays::slice::Slice::Metadata = vortex_array::arrays::slice::SliceMetadata - pub type vortex_array::arrays::slice::Slice::OperationsVTable = vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arrays::slice::Slice @@ -3942,25 +3626,19 @@ pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::slice::SliceMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::slice::Slice::len(array: &vortex_array::arrays::slice::SliceData) -> usize +pub fn vortex_array::arrays::slice::Slice::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::slice::Slice::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3970,15 +3648,13 @@ pub fn vortex_array::arrays::slice::Slice::reduce(array: vortex_array::ArrayView pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::slice::Slice::vtable(_array: &vortex_array::arrays::slice::SliceData) -> &Self +pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3990,12 +3666,6 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::slice::S pub fn vortex_array::arrays::slice::Slice::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::slice::SliceArrayParts - -pub vortex_array::arrays::slice::SliceArrayParts::child: vortex_array::ArrayRef - -pub vortex_array::arrays::slice::SliceArrayParts::range: core::ops::range::Range - pub struct vortex_array::arrays::slice::SliceData impl vortex_array::arrays::slice::SliceData @@ -4004,6 +3674,8 @@ pub fn vortex_array::arrays::slice::SliceData::child(&self) -> &vortex_array::Ar pub fn vortex_array::arrays::slice::SliceData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::slice::SliceData::into_parts(self) -> vortex_array::arrays::slice::SliceDataParts + pub fn vortex_array::arrays::slice::SliceData::is_empty(&self) -> bool pub fn vortex_array::arrays::slice::SliceData::len(&self) -> usize @@ -4012,27 +3684,19 @@ pub fn vortex_array::arrays::slice::SliceData::new(child: vortex_array::ArrayRef pub fn vortex_array::arrays::slice::SliceData::slice_range(&self) -> &core::ops::range::Range -pub fn vortex_array::arrays::slice::SliceData::try_new(child: vortex_array::ArrayRef, range: core::ops::range::Range) -> vortex_error::VortexResult - -impl vortex_array::arrays::slice::SliceData - -pub fn vortex_array::arrays::slice::SliceData::into_parts(self) -> vortex_array::arrays::slice::SliceArrayParts - impl core::clone::Clone for vortex_array::arrays::slice::SliceData pub fn vortex_array::arrays::slice::SliceData::clone(&self) -> vortex_array::arrays::slice::SliceData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::slice::SliceData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::slice::SliceData pub fn vortex_array::arrays::slice::SliceData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::slice::SliceData +pub struct vortex_array::arrays::slice::SliceDataParts + +pub vortex_array::arrays::slice::SliceDataParts::child: vortex_array::ArrayRef -pub fn vortex_array::arrays::slice::SliceData::into_array(self) -> vortex_array::ArrayRef +pub vortex_array::arrays::slice::SliceDataParts::range: core::ops::range::Range pub struct vortex_array::arrays::slice::SliceExecuteAdaptor(pub V) @@ -4174,8 +3838,6 @@ impl vortex_array::VTable for vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ArrayData = vortex_array::arrays::struct_::StructData -pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct @@ -4190,25 +3852,19 @@ pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Struct::dtype(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Struct::len(array: &vortex_array::arrays::struct_::StructData) -> usize +pub fn vortex_array::arrays::Struct::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Struct::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -4218,15 +3874,13 @@ pub fn vortex_array::arrays::Struct::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Struct::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Struct::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -4254,25 +3908,17 @@ impl vortex_array::scalar_fn::fns::zip::ZipKernel for vortex_array::arrays::Stru pub fn vortex_array::arrays::Struct::zip(if_true: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, if_false: &vortex_array::ArrayRef, mask: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::struct_::StructArrayParts - -pub vortex_array::arrays::struct_::StructArrayParts::fields: alloc::sync::Arc<[vortex_array::ArrayRef]> - -pub vortex_array::arrays::struct_::StructArrayParts::struct_fields: vortex_array::dtype::StructFields - -pub vortex_array::arrays::struct_::StructArrayParts::validity: vortex_array::validity::Validity - pub struct vortex_array::arrays::struct_::StructData impl vortex_array::arrays::struct_::StructData -pub fn vortex_array::arrays::struct_::StructData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::struct_::StructData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::struct_::StructData::from_fields>(items: &[(N, vortex_array::ArrayRef)]) -> vortex_error::VortexResult pub fn vortex_array::arrays::struct_::StructData::into_fields(self) -> alloc::vec::Vec -pub fn vortex_array::arrays::struct_::StructData::into_parts(self) -> vortex_array::arrays::struct_::StructArrayParts +pub fn vortex_array::arrays::struct_::StructData::into_parts(self) -> vortex_array::arrays::struct_::StructDataParts pub fn vortex_array::arrays::struct_::StructData::is_empty(&self) -> bool @@ -4288,20 +3934,14 @@ pub fn vortex_array::arrays::struct_::StructData::new_fieldless_with_len(len: us pub unsafe fn vortex_array::arrays::struct_::StructData::new_unchecked(fields: impl core::convert::Into>, dtype: vortex_array::dtype::StructFields, length: usize, validity: vortex_array::validity::Validity) -> Self -pub fn vortex_array::arrays::struct_::StructData::project(&self, projection: &[vortex_array::dtype::FieldName]) -> vortex_error::VortexResult - pub fn vortex_array::arrays::struct_::StructData::remove_column(&mut self, name: impl core::convert::Into) -> core::option::Option -pub fn vortex_array::arrays::struct_::StructData::struct_fields(&self) -> &vortex_array::dtype::StructFields +pub fn vortex_array::arrays::struct_::StructData::struct_fields(&self) -> vortex_array::dtype::StructFields pub fn vortex_array::arrays::struct_::StructData::try_from_iter, A: vortex_array::IntoArray, T: core::iter::traits::collect::IntoIterator>(iter: T) -> vortex_error::VortexResult pub fn vortex_array::arrays::struct_::StructData::try_from_iter_with_validity, A: vortex_array::IntoArray, T: core::iter::traits::collect::IntoIterator>(iter: T, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult -pub fn vortex_array::arrays::struct_::StructData::try_new(names: vortex_array::dtype::FieldNames, fields: impl core::convert::Into>, length: usize, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::struct_::StructData::try_new_with_dtype(fields: impl core::convert::Into>, dtype: vortex_array::dtype::StructFields, length: usize, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult - pub fn vortex_array::arrays::struct_::StructData::unmasked_field(&self, idx: usize) -> &vortex_array::ArrayRef pub fn vortex_array::arrays::struct_::StructData::unmasked_field_by_name(&self, name: impl core::convert::AsRef) -> vortex_error::VortexResult<&vortex_array::ArrayRef> @@ -4322,17 +3962,17 @@ impl core::clone::Clone for vortex_array::arrays::struct_::StructData pub fn vortex_array::arrays::struct_::StructData::clone(&self) -> vortex_array::arrays::struct_::StructData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::struct_::StructData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::struct_::StructData pub fn vortex_array::arrays::struct_::StructData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::struct_::StructData +pub struct vortex_array::arrays::struct_::StructDataParts + +pub vortex_array::arrays::struct_::StructDataParts::fields: alloc::sync::Arc<[vortex_array::ArrayRef]> + +pub vortex_array::arrays::struct_::StructDataParts::struct_fields: vortex_array::dtype::StructFields -pub fn vortex_array::arrays::struct_::StructData::into_array(self) -> vortex_array::ArrayRef +pub vortex_array::arrays::struct_::StructDataParts::validity: vortex_array::validity::Validity pub type vortex_array::arrays::struct_::StructArray = vortex_array::Array @@ -4390,8 +4030,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ArrayData = vortex_array::arrays::varbin::VarBinData -pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::VarBin::OperationsVTable = vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::VarBin @@ -4406,25 +4044,19 @@ pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBin::dtype(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::VarBin::len(array: &vortex_array::arrays::varbin::VarBinData) -> usize +pub fn vortex_array::arrays::VarBin::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::VarBin::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -4434,15 +4066,13 @@ pub fn vortex_array::arrays::VarBin::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBin::vtable(_array: &vortex_array::arrays::varbin::VarBinData) -> &Self +pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -4482,7 +4112,7 @@ pub fn vortex_array::arrays::varbin::VarBinData::bytes(&self) -> &vortex_buffer: pub fn vortex_array::arrays::varbin::VarBinData::bytes_handle(&self) -> &vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::varbin::VarBinData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::varbin::VarBinData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::varbin::VarBinData::from_iter, I: core::iter::traits::collect::IntoIterator>>(iter: I, dtype: vortex_array::dtype::DType) -> Self @@ -4560,10 +4190,6 @@ impl core::convert::From>>) -> Self -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::varbin::VarBinData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::varbin::VarBinData pub fn vortex_array::arrays::varbin::VarBinData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result @@ -4576,10 +4202,6 @@ impl core::iter::traits::collect::FromIterator>>>(iter: T) -> Self -impl vortex_array::IntoArray for vortex_array::arrays::varbin::VarBinData - -pub fn vortex_array::arrays::varbin::VarBinData::into_array(self) -> vortex_array::ArrayRef - impl<'a> core::iter::traits::collect::FromIterator> for vortex_array::arrays::varbin::VarBinData pub fn vortex_array::arrays::varbin::VarBinData::from_iter>>(iter: T) -> Self @@ -4798,8 +4420,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ArrayData = vortex_array::arrays::varbinview::VarBinViewData -pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView @@ -4814,25 +4434,19 @@ pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBinView::dtype(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::VarBinView::len(array: &vortex_array::arrays::varbinview::VarBinViewData) -> usize +pub fn vortex_array::arrays::VarBinView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::VarBinView::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -4842,15 +4456,13 @@ pub fn vortex_array::arrays::VarBinView::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBinView::vtable(_array: &vortex_array::arrays::varbinview::VarBinViewData) -> &Self +pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -4878,16 +4490,6 @@ impl vortex_array::scalar_fn::fns::zip::ZipKernel for vortex_array::arrays::VarB pub fn vortex_array::arrays::VarBinView::zip(if_true: vortex_array::ArrayView<'_, vortex_array::arrays::VarBinView>, if_false: &vortex_array::ArrayRef, mask: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::varbinview::VarBinViewArrayParts - -pub vortex_array::arrays::varbinview::VarBinViewArrayParts::buffers: alloc::sync::Arc<[vortex_array::buffer::BufferHandle]> - -pub vortex_array::arrays::varbinview::VarBinViewArrayParts::dtype: vortex_array::dtype::DType - -pub vortex_array::arrays::varbinview::VarBinViewArrayParts::validity: vortex_array::validity::Validity - -pub vortex_array::arrays::varbinview::VarBinViewArrayParts::views: vortex_array::buffer::BufferHandle - pub struct vortex_array::arrays::varbinview::VarBinViewData impl vortex_array::arrays::varbinview::VarBinViewData @@ -4898,7 +4500,7 @@ pub fn vortex_array::arrays::varbinview::VarBinViewData::bytes_at(&self, index: pub fn vortex_array::arrays::varbinview::VarBinViewData::data_buffers(&self) -> &alloc::sync::Arc<[vortex_array::buffer::BufferHandle]> -pub fn vortex_array::arrays::varbinview::VarBinViewData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::varbinview::VarBinViewData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter, I: core::iter::traits::collect::IntoIterator>>(iter: I, dtype: vortex_array::dtype::DType) -> Self @@ -4910,7 +4512,7 @@ pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter_nullable_str< pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter_str, I: core::iter::traits::collect::IntoIterator>(iter: I) -> Self -pub fn vortex_array::arrays::varbinview::VarBinViewData::into_parts(self) -> vortex_array::arrays::varbinview::VarBinViewArrayParts +pub fn vortex_array::arrays::varbinview::VarBinViewData::into_parts(self) -> vortex_array::arrays::varbinview::VarBinViewDataParts pub fn vortex_array::arrays::varbinview::VarBinViewData::is_empty(&self) -> bool @@ -4942,10 +4544,6 @@ impl core::clone::Clone for vortex_array::arrays::varbinview::VarBinViewData pub fn vortex_array::arrays::varbinview::VarBinViewData::clone(&self) -> vortex_array::arrays::varbinview::VarBinViewData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::varbinview::VarBinViewData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::varbinview::VarBinViewData pub fn vortex_array::arrays::varbinview::VarBinViewData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result @@ -4958,10 +4556,6 @@ impl core::iter::traits::collect::FromIterator>>>(iter: T) -> Self -impl vortex_array::IntoArray for vortex_array::arrays::varbinview::VarBinViewData - -pub fn vortex_array::arrays::varbinview::VarBinViewData::into_array(self) -> vortex_array::ArrayRef - impl<'a> core::iter::traits::collect::FromIterator> for vortex_array::arrays::varbinview::VarBinViewData pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter>>(iter: T) -> Self @@ -4970,6 +4564,16 @@ impl<'a> core::iter::traits::collect::FromIterator pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter>>(iter: T) -> Self +pub struct vortex_array::arrays::varbinview::VarBinViewDataParts + +pub vortex_array::arrays::varbinview::VarBinViewDataParts::buffers: alloc::sync::Arc<[vortex_array::buffer::BufferHandle]> + +pub vortex_array::arrays::varbinview::VarBinViewDataParts::dtype: vortex_array::dtype::DType + +pub vortex_array::arrays::varbinview::VarBinViewDataParts::validity: vortex_array::validity::Validity + +pub vortex_array::arrays::varbinview::VarBinViewDataParts::views: vortex_array::buffer::BufferHandle + pub type vortex_array::arrays::varbinview::VarBinViewArray = vortex_array::Array pub mod vortex_array::arrays::variant @@ -4996,8 +4600,6 @@ impl vortex_array::VTable for vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ArrayData = vortex_array::arrays::variant::VariantData -pub type vortex_array::arrays::Variant::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Variant::OperationsVTable = vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::Variant @@ -5012,25 +4614,19 @@ pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Variant::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Variant::len(array: &Self::ArrayData) -> usize +pub fn vortex_array::arrays::Variant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Variant::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5040,15 +4636,13 @@ pub fn vortex_array::arrays::Variant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Variant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5062,7 +4656,7 @@ impl vortex_array::arrays::variant::VariantData pub fn vortex_array::arrays::variant::VariantData::child(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::variant::VariantData::dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::variant::VariantData::dtype(&self) -> vortex_array::dtype::DType pub fn vortex_array::arrays::variant::VariantData::is_empty(&self) -> bool @@ -5074,18 +4668,10 @@ impl core::clone::Clone for vortex_array::arrays::variant::VariantData pub fn vortex_array::arrays::variant::VariantData::clone(&self) -> vortex_array::arrays::variant::VariantData -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::variant::VariantData) -> vortex_array::ArrayRef - impl core::fmt::Debug for vortex_array::arrays::variant::VariantData pub fn vortex_array::arrays::variant::VariantData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::IntoArray for vortex_array::arrays::variant::VariantData - -pub fn vortex_array::arrays::variant::VariantData::into_array(self) -> vortex_array::ArrayRef - pub type vortex_array::arrays::variant::VariantArray = vortex_array::Array pub struct vortex_array::arrays::Bool @@ -5110,8 +4696,6 @@ impl vortex_array::VTable for vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ArrayData = vortex_array::arrays::bool::BoolData -pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Bool::OperationsVTable = vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool @@ -5126,25 +4710,19 @@ pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Bool::dtype(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Bool::len(array: &vortex_array::arrays::bool::BoolData) -> usize +pub fn vortex_array::arrays::Bool::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Bool::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5154,15 +4732,13 @@ pub fn vortex_array::arrays::Bool::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Bool::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5222,8 +4798,6 @@ impl vortex_array::VTable for vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ArrayData = vortex_array::arrays::chunked::ChunkedData -pub type vortex_array::arrays::Chunked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Chunked::OperationsVTable = vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::Chunked @@ -5238,25 +4812,19 @@ pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Chunked::dtype(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Chunked::len(array: &vortex_array::arrays::chunked::ChunkedData) -> usize +pub fn vortex_array::arrays::Chunked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Chunked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5266,15 +4834,13 @@ pub fn vortex_array::arrays::Chunked::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Chunked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5336,8 +4902,6 @@ impl vortex_array::VTable for vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ArrayData = vortex_array::arrays::constant::ConstantData -pub type vortex_array::arrays::Constant::Metadata = vortex_array::scalar::Scalar - pub type vortex_array::arrays::Constant::OperationsVTable = vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays::Constant @@ -5352,25 +4916,19 @@ pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Constant::dtype(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Constant::len(array: &vortex_array::arrays::constant::ConstantData) -> usize +pub fn vortex_array::arrays::Constant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Constant::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5380,15 +4938,13 @@ pub fn vortex_array::arrays::Constant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Constant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5446,8 +5002,6 @@ impl vortex_array::VTable for vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ArrayData = vortex_array::arrays::decimal::DecimalData -pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Decimal::OperationsVTable = vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::Decimal @@ -5462,25 +5016,19 @@ pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Decimal::dtype(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Decimal::len(array: &vortex_array::arrays::decimal::DecimalData) -> usize +pub fn vortex_array::arrays::Decimal::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Decimal::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5490,15 +5038,13 @@ pub fn vortex_array::arrays::Decimal::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5558,8 +5104,6 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData -pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -5574,25 +5118,19 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize +pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5602,15 +5140,13 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5672,8 +5208,6 @@ impl vortex_array::VTable for vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData -pub type vortex_array::arrays::Extension::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -5688,25 +5222,19 @@ pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Extension::dtype(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Extension::len(array: &vortex_array::arrays::extension::ExtensionData) -> usize +pub fn vortex_array::arrays::Extension::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Extension::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5716,15 +5244,13 @@ pub fn vortex_array::arrays::Extension::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Extension::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Extension::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5778,8 +5304,6 @@ impl vortex_array::VTable for vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ArrayData = vortex_array::arrays::filter::FilterData -pub type vortex_array::arrays::Filter::Metadata = vortex_array::arrays::filter::vtable::FilterMetadata - pub type vortex_array::arrays::Filter::OperationsVTable = vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Filter @@ -5794,25 +5318,19 @@ pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::filter::vtable::FilterMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Filter::dtype(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Filter::len(array: &vortex_array::arrays::filter::FilterData) -> usize +pub fn vortex_array::arrays::Filter::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Filter::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5822,15 +5340,13 @@ pub fn vortex_array::arrays::Filter::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Filter::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Filter::vtable(_array: &vortex_array::arrays::filter::FilterData) -> &Self +pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5860,8 +5376,6 @@ impl vortex_array::VTable for vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ArrayData = vortex_array::arrays::fixed_size_list::FixedSizeListData -pub type vortex_array::arrays::FixedSizeList::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::FixedSizeList::OperationsVTable = vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arrays::FixedSizeList @@ -5876,25 +5390,19 @@ pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::FixedSizeList::dtype(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::FixedSizeList::len(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> usize +pub fn vortex_array::arrays::FixedSizeList::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::FixedSizeList::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5904,15 +5412,13 @@ pub fn vortex_array::arrays::FixedSizeList::reduce(array: vortex_array::ArrayVie pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::FixedSizeList::vtable(_array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &Self +pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5958,8 +5464,6 @@ impl vortex_array::VTable for vortex_array::arrays::List pub type vortex_array::arrays::List::ArrayData = vortex_array::arrays::list::ListData -pub type vortex_array::arrays::List::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::List::OperationsVTable = vortex_array::arrays::List pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List @@ -5974,25 +5478,19 @@ pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::List::dtype(array: &vortex_array::arrays::list::ListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::List::len(array: &vortex_array::arrays::list::ListData) -> usize +pub fn vortex_array::arrays::List::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::List::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6002,15 +5500,13 @@ pub fn vortex_array::arrays::List::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::List::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::list::ListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::List::vtable(_array: &vortex_array::arrays::list::ListData) -> &Self +pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6060,8 +5556,6 @@ impl vortex_array::VTable for vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ArrayData = vortex_array::arrays::listview::ListViewData -pub type vortex_array::arrays::ListView::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::ListView::OperationsVTable = vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays::ListView @@ -6076,25 +5570,19 @@ pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::ListView::dtype(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::ListView::len(array: &vortex_array::arrays::listview::ListViewData) -> usize +pub fn vortex_array::arrays::ListView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::ListView::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6104,15 +5592,13 @@ pub fn vortex_array::arrays::ListView::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::ListView::vtable(_array: &vortex_array::arrays::listview::ListViewData) -> &Self +pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6158,8 +5644,6 @@ impl vortex_array::VTable for vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ArrayData = vortex_array::arrays::masked::MaskedData -pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked @@ -6174,25 +5658,19 @@ pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Masked::dtype(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Masked::len(array: &vortex_array::arrays::masked::MaskedData) -> usize +pub fn vortex_array::arrays::Masked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Masked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6202,15 +5680,13 @@ pub fn vortex_array::arrays::Masked::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Masked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Masked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6260,8 +5736,6 @@ impl vortex_array::VTable for vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ArrayData = vortex_array::arrays::null::NullData -pub type vortex_array::arrays::null::Null::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::null::Null::OperationsVTable = vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays::null::Null @@ -6276,25 +5750,19 @@ pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::null::Null::dtype(_array: &vortex_array::arrays::null::NullData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::null::Null::len(array: &vortex_array::arrays::null::NullData) -> usize +pub fn vortex_array::arrays::null::Null::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::null::Null::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6304,15 +5772,13 @@ pub fn vortex_array::arrays::null::Null::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::null::Null::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::null::Null::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6358,8 +5824,6 @@ impl vortex_array::VTable for vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray -pub type vortex_array::arrays::patched::Patched::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -6374,25 +5838,19 @@ pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::patched::Patched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::patched::Patched::len(array: &Self::ArrayData) -> usize +pub fn vortex_array::arrays::patched::Patched::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::patched::Patched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6402,15 +5860,13 @@ pub fn vortex_array::arrays::patched::Patched::reduce(array: vortex_array::Array pub fn vortex_array::arrays::patched::Patched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::patched::Patched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::patched::Patched::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6440,8 +5896,12 @@ impl vortex_array::arrays::patched::PatchedArray pub fn vortex_array::arrays::patched::PatchedArray::base_array(&self) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::PatchedArray::is_empty(&self) -> bool + pub fn vortex_array::arrays::patched::PatchedArray::lane_offsets(&self) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::PatchedArray::len(&self) -> usize + pub fn vortex_array::arrays::patched::PatchedArray::patch_indices(&self) -> &vortex_array::ArrayRef pub fn vortex_array::arrays::patched::PatchedArray::patch_values(&self) -> &vortex_array::ArrayRef @@ -6452,7 +5912,7 @@ pub fn vortex_array::arrays::patched::PatchedArray::from_array_and_patches(inner impl vortex_array::arrays::patched::PatchedArray -pub fn vortex_array::arrays::patched::PatchedArray::to_array(&self) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::patched::PatchedArray::to_array(&self) -> vortex_array::ArrayRef where Self: core::clone::Clone + vortex_array::IntoArray impl core::clone::Clone for vortex_array::arrays::patched::PatchedArray @@ -6460,7 +5920,7 @@ pub fn vortex_array::arrays::patched::PatchedArray::clone(&self) -> vortex_array impl core::convert::From for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> vortex_array::ArrayRef +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> Self impl core::fmt::Debug for vortex_array::arrays::patched::PatchedArray @@ -6492,8 +5952,6 @@ impl vortex_array::VTable for vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ArrayData = vortex_array::arrays::primitive::PrimitiveData -pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive @@ -6508,25 +5966,19 @@ pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Primitive::dtype(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Primitive::len(array: &vortex_array::arrays::primitive::PrimitiveData) -> usize +pub fn vortex_array::arrays::Primitive::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Primitive::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6536,15 +5988,13 @@ pub fn vortex_array::arrays::Primitive::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6600,8 +6050,6 @@ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::Metadata = vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata - pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -6616,25 +6064,19 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::dtype(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::len(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> usize +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6644,15 +6086,13 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_arr pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::vtable(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &Self +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6682,8 +6122,6 @@ impl vortex_array::VTable for vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ArrayData = vortex_array::arrays::shared::SharedData -pub type vortex_array::arrays::Shared::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Shared::OperationsVTable = vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Shared @@ -6698,25 +6136,19 @@ pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Shared::dtype(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Shared::len(array: &vortex_array::arrays::shared::SharedData) -> usize +pub fn vortex_array::arrays::Shared::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Shared::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6726,15 +6158,13 @@ pub fn vortex_array::arrays::Shared::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Shared::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Shared::vtable(_array: &vortex_array::arrays::shared::SharedData) -> &Self +pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6764,8 +6194,6 @@ impl vortex_array::VTable for vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ArrayData = vortex_array::arrays::slice::SliceData -pub type vortex_array::arrays::slice::Slice::Metadata = vortex_array::arrays::slice::SliceMetadata - pub type vortex_array::arrays::slice::Slice::OperationsVTable = vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arrays::slice::Slice @@ -6780,25 +6208,19 @@ pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::slice::SliceMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::slice::Slice::len(array: &vortex_array::arrays::slice::SliceData) -> usize +pub fn vortex_array::arrays::slice::Slice::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::slice::Slice::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6808,15 +6230,13 @@ pub fn vortex_array::arrays::slice::Slice::reduce(array: vortex_array::ArrayView pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::slice::Slice::vtable(_array: &vortex_array::arrays::slice::SliceData) -> &Self +pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6850,8 +6270,6 @@ impl vortex_array::VTable for vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ArrayData = vortex_array::arrays::struct_::StructData -pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct @@ -6866,25 +6284,19 @@ pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Struct::dtype(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Struct::len(array: &vortex_array::arrays::struct_::StructData) -> usize +pub fn vortex_array::arrays::Struct::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Struct::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6894,15 +6306,13 @@ pub fn vortex_array::arrays::Struct::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Struct::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Struct::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6956,8 +6366,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ArrayData = vortex_array::arrays::varbin::VarBinData -pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::VarBin::OperationsVTable = vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::VarBin @@ -6972,25 +6380,19 @@ pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBin::dtype(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::VarBin::len(array: &vortex_array::arrays::varbin::VarBinData) -> usize +pub fn vortex_array::arrays::VarBin::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::VarBin::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -7000,15 +6402,13 @@ pub fn vortex_array::arrays::VarBin::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBin::vtable(_array: &vortex_array::arrays::varbin::VarBinData) -> &Self +pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -7062,8 +6462,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ArrayData = vortex_array::arrays::varbinview::VarBinViewData -pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView @@ -7078,25 +6476,19 @@ pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBinView::dtype(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::VarBinView::len(array: &vortex_array::arrays::varbinview::VarBinViewData) -> usize +pub fn vortex_array::arrays::VarBinView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::VarBinView::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -7106,15 +6498,13 @@ pub fn vortex_array::arrays::VarBinView::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBinView::vtable(_array: &vortex_array::arrays::varbinview::VarBinViewData) -> &Self +pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -7164,8 +6554,6 @@ impl vortex_array::VTable for vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ArrayData = vortex_array::arrays::variant::VariantData -pub type vortex_array::arrays::Variant::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Variant::OperationsVTable = vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::Variant @@ -7180,25 +6568,19 @@ pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Variant::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Variant::len(array: &Self::ArrayData) -> usize +pub fn vortex_array::arrays::Variant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Variant::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -7208,15 +6590,13 @@ pub fn vortex_array::arrays::Variant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Variant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -7570,11 +6950,11 @@ impl core::clone::Clone for vortex_array::buffer::BufferHandle pub fn vortex_array::buffer::BufferHandle::clone(&self) -> vortex_array::buffer::BufferHandle -impl core::convert::TryFrom for vortex_array::serde::ArrayParts +impl core::convert::TryFrom for vortex_array::serde::SerializedArray -pub type vortex_array::serde::ArrayParts::Error = vortex_error::VortexError +pub type vortex_array::serde::SerializedArray::Error = vortex_error::VortexError -pub fn vortex_array::serde::ArrayParts::try_from(value: vortex_array::buffer::BufferHandle) -> core::result::Result +pub fn vortex_array::serde::SerializedArray::try_from(value: vortex_array::buffer::BufferHandle) -> core::result::Result impl core::fmt::Debug for vortex_array::buffer::BufferHandle @@ -18804,67 +18184,67 @@ pub fn vortex_array::serde::ArrayNodeFlatBuffer<'a>::try_new(ctx: &'a vortex_arr pub fn vortex_array::serde::ArrayNodeFlatBuffer<'a>::try_write_flatbuffer<'fb>(&self, fbb: &mut flatbuffers::builder::FlatBufferBuilder<'fb>) -> vortex_error::VortexResult>> -pub struct vortex_array::serde::ArrayParts +pub struct vortex_array::serde::SerializeOptions -impl vortex_array::serde::ArrayParts +pub vortex_array::serde::SerializeOptions::include_padding: bool -pub fn vortex_array::serde::ArrayParts::buffer(&self, idx: usize) -> vortex_error::VortexResult +pub vortex_array::serde::SerializeOptions::offset: usize -pub fn vortex_array::serde::ArrayParts::buffer_lengths(&self) -> alloc::vec::Vec +impl core::default::Default for vortex_array::serde::SerializeOptions -pub fn vortex_array::serde::ArrayParts::child(&self, idx: usize) -> vortex_array::serde::ArrayParts +pub fn vortex_array::serde::SerializeOptions::default() -> vortex_array::serde::SerializeOptions -pub fn vortex_array::serde::ArrayParts::decode(&self, dtype: &vortex_array::dtype::DType, len: usize, ctx: &vortex_session::registry::ReadContext, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +impl core::fmt::Debug for vortex_array::serde::SerializeOptions -pub fn vortex_array::serde::ArrayParts::encoding_id(&self) -> u16 +pub fn vortex_array::serde::SerializeOptions::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::serde::ArrayParts::from_array_tree(array_tree: impl core::convert::Into) -> vortex_error::VortexResult +pub struct vortex_array::serde::SerializedArray -pub fn vortex_array::serde::ArrayParts::from_flatbuffer_and_segment(array_tree: vortex_buffer::ByteBuffer, segment: vortex_array::buffer::BufferHandle) -> vortex_error::VortexResult +impl vortex_array::serde::SerializedArray -pub fn vortex_array::serde::ArrayParts::from_flatbuffer_and_segment_with_overrides(array_tree: vortex_buffer::ByteBuffer, segment: vortex_array::buffer::BufferHandle, buffer_overrides: &vortex_utils::aliases::hash_map::HashMap) -> vortex_error::VortexResult +pub fn vortex_array::serde::SerializedArray::buffer(&self, idx: usize) -> vortex_error::VortexResult -pub fn vortex_array::serde::ArrayParts::from_flatbuffer_with_buffers(array_tree: impl core::convert::Into, buffers: alloc::vec::Vec) -> vortex_error::VortexResult +pub fn vortex_array::serde::SerializedArray::buffer_lengths(&self) -> alloc::vec::Vec -pub fn vortex_array::serde::ArrayParts::metadata(&self) -> &[u8] +pub fn vortex_array::serde::SerializedArray::child(&self, idx: usize) -> vortex_array::serde::SerializedArray -pub fn vortex_array::serde::ArrayParts::nbuffers(&self) -> usize +pub fn vortex_array::serde::SerializedArray::decode(&self, dtype: &vortex_array::dtype::DType, len: usize, ctx: &vortex_session::registry::ReadContext, session: &vortex_session::VortexSession) -> vortex_error::VortexResult -pub fn vortex_array::serde::ArrayParts::nchildren(&self) -> usize +pub fn vortex_array::serde::SerializedArray::encoding_id(&self) -> u16 -impl core::clone::Clone for vortex_array::serde::ArrayParts +pub fn vortex_array::serde::SerializedArray::from_array_tree(array_tree: impl core::convert::Into) -> vortex_error::VortexResult -pub fn vortex_array::serde::ArrayParts::clone(&self) -> vortex_array::serde::ArrayParts +pub fn vortex_array::serde::SerializedArray::from_flatbuffer_and_segment(array_tree: vortex_buffer::ByteBuffer, segment: vortex_array::buffer::BufferHandle) -> vortex_error::VortexResult -impl core::convert::TryFrom for vortex_array::serde::ArrayParts +pub fn vortex_array::serde::SerializedArray::from_flatbuffer_and_segment_with_overrides(array_tree: vortex_buffer::ByteBuffer, segment: vortex_array::buffer::BufferHandle, buffer_overrides: &vortex_utils::aliases::hash_map::HashMap) -> vortex_error::VortexResult -pub type vortex_array::serde::ArrayParts::Error = vortex_error::VortexError +pub fn vortex_array::serde::SerializedArray::from_flatbuffer_with_buffers(array_tree: impl core::convert::Into, buffers: alloc::vec::Vec) -> vortex_error::VortexResult -pub fn vortex_array::serde::ArrayParts::try_from(value: vortex_array::buffer::BufferHandle) -> core::result::Result +pub fn vortex_array::serde::SerializedArray::metadata(&self) -> &[u8] -impl core::convert::TryFrom> for vortex_array::serde::ArrayParts +pub fn vortex_array::serde::SerializedArray::nbuffers(&self) -> usize -pub type vortex_array::serde::ArrayParts::Error = vortex_error::VortexError +pub fn vortex_array::serde::SerializedArray::nchildren(&self) -> usize -pub fn vortex_array::serde::ArrayParts::try_from(value: vortex_buffer::ByteBuffer) -> core::result::Result +impl core::clone::Clone for vortex_array::serde::SerializedArray -impl core::fmt::Debug for vortex_array::serde::ArrayParts +pub fn vortex_array::serde::SerializedArray::clone(&self) -> vortex_array::serde::SerializedArray -pub fn vortex_array::serde::ArrayParts::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::convert::TryFrom for vortex_array::serde::SerializedArray -pub struct vortex_array::serde::SerializeOptions +pub type vortex_array::serde::SerializedArray::Error = vortex_error::VortexError -pub vortex_array::serde::SerializeOptions::include_padding: bool +pub fn vortex_array::serde::SerializedArray::try_from(value: vortex_array::buffer::BufferHandle) -> core::result::Result -pub vortex_array::serde::SerializeOptions::offset: usize +impl core::convert::TryFrom> for vortex_array::serde::SerializedArray -impl core::default::Default for vortex_array::serde::SerializeOptions +pub type vortex_array::serde::SerializedArray::Error = vortex_error::VortexError -pub fn vortex_array::serde::SerializeOptions::default() -> vortex_array::serde::SerializeOptions +pub fn vortex_array::serde::SerializedArray::try_from(value: vortex_buffer::ByteBuffer) -> core::result::Result -impl core::fmt::Debug for vortex_array::serde::SerializeOptions +impl core::fmt::Debug for vortex_array::serde::SerializedArray -pub fn vortex_array::serde::SerializeOptions::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_array::serde::SerializedArray::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub trait vortex_array::serde::ArrayChildren @@ -19408,9 +18788,7 @@ pub fn vortex_array::ValidityVTableFromValiditySliceHelper::validity(array: vort pub trait vortex_array::vtable::ArrayVTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug -pub type vortex_array::vtable::ArrayVTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug + vortex_array::IntoArray - -pub type vortex_array::vtable::ArrayVTable::Metadata: core::fmt::Debug +pub type vortex_array::vtable::ArrayVTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug pub type vortex_array::vtable::ArrayVTable::OperationsVTable: vortex_array::OperationsVTable @@ -19426,25 +18804,19 @@ pub fn vortex_array::vtable::ArrayVTable::buffer(array: vortex_array::ArrayView< pub fn vortex_array::vtable::ArrayVTable::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::vtable::ArrayVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::vtable::ArrayVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::vtable::ArrayVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::vtable::ArrayVTable::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::vtable::ArrayVTable::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::vtable::ArrayVTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::vtable::ArrayVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::vtable::ArrayVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::vtable::ArrayVTable::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::vtable::ArrayVTable::len(array: &Self::ArrayData) -> usize +pub fn vortex_array::vtable::ArrayVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::vtable::ArrayVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::vtable::ArrayVTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::vtable::ArrayVTable::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19454,15 +18826,13 @@ pub fn vortex_array::vtable::ArrayVTable::reduce(array: vortex_array::ArrayView< pub fn vortex_array::vtable::ArrayVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::vtable::ArrayVTable::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::vtable::ArrayVTable::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::vtable::ArrayVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::vtable::ArrayVTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] -pub fn vortex_array::vtable::ArrayVTable::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::vtable::ArrayVTable::vtable(array: &Self::ArrayData) -> &Self +pub fn vortex_array::vtable::ArrayVTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::vtable::ArrayVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19470,8 +18840,6 @@ impl vortex_array::VTable for vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ArrayData = vortex_array::arrays::bool::BoolData -pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Bool::OperationsVTable = vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool @@ -19486,25 +18854,19 @@ pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Bool::dtype(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Bool::len(array: &vortex_array::arrays::bool::BoolData) -> usize +pub fn vortex_array::arrays::Bool::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Bool::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19514,15 +18876,13 @@ pub fn vortex_array::arrays::Bool::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Bool::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19530,8 +18890,6 @@ impl vortex_array::VTable for vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ArrayData = vortex_array::arrays::chunked::ChunkedData -pub type vortex_array::arrays::Chunked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Chunked::OperationsVTable = vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::Chunked @@ -19546,25 +18904,19 @@ pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Chunked::dtype(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Chunked::len(array: &vortex_array::arrays::chunked::ChunkedData) -> usize +pub fn vortex_array::arrays::Chunked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Chunked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19574,15 +18926,13 @@ pub fn vortex_array::arrays::Chunked::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Chunked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19590,8 +18940,6 @@ impl vortex_array::VTable for vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ArrayData = vortex_array::arrays::constant::ConstantData -pub type vortex_array::arrays::Constant::Metadata = vortex_array::scalar::Scalar - pub type vortex_array::arrays::Constant::OperationsVTable = vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays::Constant @@ -19606,25 +18954,19 @@ pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Constant::dtype(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Constant::len(array: &vortex_array::arrays::constant::ConstantData) -> usize +pub fn vortex_array::arrays::Constant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Constant::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19634,15 +18976,13 @@ pub fn vortex_array::arrays::Constant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Constant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19650,8 +18990,6 @@ impl vortex_array::VTable for vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ArrayData = vortex_array::arrays::decimal::DecimalData -pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Decimal::OperationsVTable = vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::Decimal @@ -19666,25 +19004,19 @@ pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Decimal::dtype(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Decimal::len(array: &vortex_array::arrays::decimal::DecimalData) -> usize +pub fn vortex_array::arrays::Decimal::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Decimal::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19694,15 +19026,13 @@ pub fn vortex_array::arrays::Decimal::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19710,8 +19040,6 @@ impl vortex_array::VTable for vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData -pub type vortex_array::arrays::Extension::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -19726,25 +19054,19 @@ pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Extension::dtype(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Extension::len(array: &vortex_array::arrays::extension::ExtensionData) -> usize +pub fn vortex_array::arrays::Extension::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Extension::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19754,15 +19076,13 @@ pub fn vortex_array::arrays::Extension::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Extension::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Extension::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19770,8 +19090,6 @@ impl vortex_array::VTable for vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ArrayData = vortex_array::arrays::filter::FilterData -pub type vortex_array::arrays::Filter::Metadata = vortex_array::arrays::filter::vtable::FilterMetadata - pub type vortex_array::arrays::Filter::OperationsVTable = vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Filter @@ -19786,25 +19104,19 @@ pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::filter::vtable::FilterMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Filter::dtype(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Filter::len(array: &vortex_array::arrays::filter::FilterData) -> usize +pub fn vortex_array::arrays::Filter::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Filter::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19814,15 +19126,13 @@ pub fn vortex_array::arrays::Filter::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Filter::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Filter::vtable(_array: &vortex_array::arrays::filter::FilterData) -> &Self +pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19830,8 +19140,6 @@ impl vortex_array::VTable for vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ArrayData = vortex_array::arrays::fixed_size_list::FixedSizeListData -pub type vortex_array::arrays::FixedSizeList::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::FixedSizeList::OperationsVTable = vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arrays::FixedSizeList @@ -19846,25 +19154,19 @@ pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::FixedSizeList::dtype(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::FixedSizeList::len(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> usize +pub fn vortex_array::arrays::FixedSizeList::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::FixedSizeList::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19874,15 +19176,13 @@ pub fn vortex_array::arrays::FixedSizeList::reduce(array: vortex_array::ArrayVie pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::FixedSizeList::vtable(_array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &Self +pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19890,8 +19190,6 @@ impl vortex_array::VTable for vortex_array::arrays::List pub type vortex_array::arrays::List::ArrayData = vortex_array::arrays::list::ListData -pub type vortex_array::arrays::List::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::List::OperationsVTable = vortex_array::arrays::List pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List @@ -19906,25 +19204,19 @@ pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::List::dtype(array: &vortex_array::arrays::list::ListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::List::len(array: &vortex_array::arrays::list::ListData) -> usize +pub fn vortex_array::arrays::List::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::List::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19934,15 +19226,13 @@ pub fn vortex_array::arrays::List::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::List::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::list::ListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::List::vtable(_array: &vortex_array::arrays::list::ListData) -> &Self +pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19950,8 +19240,6 @@ impl vortex_array::VTable for vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ArrayData = vortex_array::arrays::listview::ListViewData -pub type vortex_array::arrays::ListView::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::ListView::OperationsVTable = vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays::ListView @@ -19966,25 +19254,19 @@ pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::ListView::dtype(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::ListView::len(array: &vortex_array::arrays::listview::ListViewData) -> usize +pub fn vortex_array::arrays::ListView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::ListView::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19994,15 +19276,13 @@ pub fn vortex_array::arrays::ListView::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::ListView::vtable(_array: &vortex_array::arrays::listview::ListViewData) -> &Self +pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20010,8 +19290,6 @@ impl vortex_array::VTable for vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ArrayData = vortex_array::arrays::masked::MaskedData -pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked @@ -20026,25 +19304,19 @@ pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Masked::dtype(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Masked::len(array: &vortex_array::arrays::masked::MaskedData) -> usize +pub fn vortex_array::arrays::Masked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Masked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20054,15 +19326,13 @@ pub fn vortex_array::arrays::Masked::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Masked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Masked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20070,8 +19340,6 @@ impl vortex_array::VTable for vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ArrayData = vortex_array::arrays::primitive::PrimitiveData -pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive @@ -20086,25 +19354,19 @@ pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Primitive::dtype(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Primitive::len(array: &vortex_array::arrays::primitive::PrimitiveData) -> usize +pub fn vortex_array::arrays::Primitive::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Primitive::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20114,15 +19376,13 @@ pub fn vortex_array::arrays::Primitive::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20130,8 +19390,6 @@ impl vortex_array::VTable for vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ArrayData = vortex_array::arrays::shared::SharedData -pub type vortex_array::arrays::Shared::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Shared::OperationsVTable = vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Shared @@ -20146,25 +19404,19 @@ pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Shared::dtype(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Shared::len(array: &vortex_array::arrays::shared::SharedData) -> usize +pub fn vortex_array::arrays::Shared::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Shared::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20174,15 +19426,13 @@ pub fn vortex_array::arrays::Shared::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Shared::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Shared::vtable(_array: &vortex_array::arrays::shared::SharedData) -> &Self +pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20190,8 +19440,6 @@ impl vortex_array::VTable for vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ArrayData = vortex_array::arrays::struct_::StructData -pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct @@ -20206,25 +19454,19 @@ pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Struct::dtype(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Struct::len(array: &vortex_array::arrays::struct_::StructData) -> usize +pub fn vortex_array::arrays::Struct::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Struct::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20234,15 +19476,13 @@ pub fn vortex_array::arrays::Struct::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Struct::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Struct::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20250,8 +19490,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ArrayData = vortex_array::arrays::varbin::VarBinData -pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::VarBin::OperationsVTable = vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::VarBin @@ -20266,25 +19504,19 @@ pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBin::dtype(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::VarBin::len(array: &vortex_array::arrays::varbin::VarBinData) -> usize +pub fn vortex_array::arrays::VarBin::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::VarBin::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20294,15 +19526,13 @@ pub fn vortex_array::arrays::VarBin::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBin::vtable(_array: &vortex_array::arrays::varbin::VarBinData) -> &Self +pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20310,8 +19540,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ArrayData = vortex_array::arrays::varbinview::VarBinViewData -pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView @@ -20326,25 +19554,19 @@ pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBinView::dtype(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::VarBinView::len(array: &vortex_array::arrays::varbinview::VarBinViewData) -> usize +pub fn vortex_array::arrays::VarBinView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::VarBinView::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20354,15 +19576,13 @@ pub fn vortex_array::arrays::VarBinView::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBinView::vtable(_array: &vortex_array::arrays::varbinview::VarBinViewData) -> &Self +pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20370,8 +19590,6 @@ impl vortex_array::VTable for vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ArrayData = vortex_array::arrays::variant::VariantData -pub type vortex_array::arrays::Variant::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Variant::OperationsVTable = vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::Variant @@ -20386,25 +19604,19 @@ pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Variant::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Variant::len(array: &Self::ArrayData) -> usize +pub fn vortex_array::arrays::Variant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Variant::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20414,15 +19626,13 @@ pub fn vortex_array::arrays::Variant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Variant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20430,8 +19640,6 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData -pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -20446,25 +19654,19 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize +pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20474,15 +19676,13 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20490,8 +19690,6 @@ impl vortex_array::VTable for vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ArrayData = vortex_array::arrays::null::NullData -pub type vortex_array::arrays::null::Null::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::null::Null::OperationsVTable = vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays::null::Null @@ -20506,25 +19704,19 @@ pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::null::Null::dtype(_array: &vortex_array::arrays::null::NullData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::null::Null::len(array: &vortex_array::arrays::null::NullData) -> usize +pub fn vortex_array::arrays::null::Null::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::null::Null::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20534,15 +19726,13 @@ pub fn vortex_array::arrays::null::Null::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::null::Null::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::null::Null::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20550,8 +19740,6 @@ impl vortex_array::VTable for vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray -pub type vortex_array::arrays::patched::Patched::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -20566,25 +19754,19 @@ pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::patched::Patched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::patched::Patched::len(array: &Self::ArrayData) -> usize +pub fn vortex_array::arrays::patched::Patched::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::patched::Patched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20594,15 +19776,13 @@ pub fn vortex_array::arrays::patched::Patched::reduce(array: vortex_array::Array pub fn vortex_array::arrays::patched::Patched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::patched::Patched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::patched::Patched::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20610,8 +19790,6 @@ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::Metadata = vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata - pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -20626,25 +19804,19 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::dtype(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::len(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> usize +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20654,15 +19826,13 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_arr pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::vtable(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &Self +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20670,8 +19840,6 @@ impl vortex_array::VTable for vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ArrayData = vortex_array::arrays::slice::SliceData -pub type vortex_array::arrays::slice::Slice::Metadata = vortex_array::arrays::slice::SliceMetadata - pub type vortex_array::arrays::slice::Slice::OperationsVTable = vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arrays::slice::Slice @@ -20686,25 +19854,19 @@ pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::slice::SliceMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::slice::Slice::len(array: &vortex_array::arrays::slice::SliceData) -> usize +pub fn vortex_array::arrays::slice::Slice::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::slice::Slice::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20714,15 +19876,13 @@ pub fn vortex_array::arrays::slice::Slice::reduce(array: vortex_array::ArrayView pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::slice::Slice::vtable(_array: &vortex_array::arrays::slice::SliceData) -> &Self +pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20852,9 +20012,7 @@ pub fn vortex_array::NotSupported::scalar_at(array: vortex_array::ArrayView<'_, pub trait vortex_array::vtable::VTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug -pub type vortex_array::vtable::VTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug + vortex_array::IntoArray - -pub type vortex_array::vtable::VTable::Metadata: core::fmt::Debug +pub type vortex_array::vtable::VTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug pub type vortex_array::vtable::VTable::OperationsVTable: vortex_array::OperationsVTable @@ -20870,25 +20028,19 @@ pub fn vortex_array::vtable::VTable::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::vtable::VTable::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::vtable::VTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::vtable::VTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::vtable::VTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::vtable::VTable::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::vtable::VTable::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::vtable::VTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::vtable::VTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::vtable::VTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::vtable::VTable::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::vtable::VTable::len(array: &Self::ArrayData) -> usize +pub fn vortex_array::vtable::VTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::vtable::VTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::vtable::VTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::vtable::VTable::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20898,15 +20050,13 @@ pub fn vortex_array::vtable::VTable::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::vtable::VTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::vtable::VTable::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::vtable::VTable::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::vtable::VTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::vtable::VTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] -pub fn vortex_array::vtable::VTable::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::vtable::VTable::vtable(array: &Self::ArrayData) -> &Self +pub fn vortex_array::vtable::VTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::vtable::VTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20914,8 +20064,6 @@ impl vortex_array::VTable for vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ArrayData = vortex_array::arrays::bool::BoolData -pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Bool::OperationsVTable = vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool @@ -20930,25 +20078,19 @@ pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Bool::dtype(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Bool::len(array: &vortex_array::arrays::bool::BoolData) -> usize +pub fn vortex_array::arrays::Bool::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Bool::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20958,15 +20100,13 @@ pub fn vortex_array::arrays::Bool::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Bool::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20974,8 +20114,6 @@ impl vortex_array::VTable for vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ArrayData = vortex_array::arrays::chunked::ChunkedData -pub type vortex_array::arrays::Chunked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Chunked::OperationsVTable = vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::Chunked @@ -20990,25 +20128,19 @@ pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Chunked::dtype(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Chunked::len(array: &vortex_array::arrays::chunked::ChunkedData) -> usize +pub fn vortex_array::arrays::Chunked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Chunked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21018,15 +20150,13 @@ pub fn vortex_array::arrays::Chunked::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Chunked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21034,8 +20164,6 @@ impl vortex_array::VTable for vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ArrayData = vortex_array::arrays::constant::ConstantData -pub type vortex_array::arrays::Constant::Metadata = vortex_array::scalar::Scalar - pub type vortex_array::arrays::Constant::OperationsVTable = vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays::Constant @@ -21050,25 +20178,19 @@ pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Constant::dtype(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Constant::len(array: &vortex_array::arrays::constant::ConstantData) -> usize +pub fn vortex_array::arrays::Constant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Constant::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21078,15 +20200,13 @@ pub fn vortex_array::arrays::Constant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Constant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21094,8 +20214,6 @@ impl vortex_array::VTable for vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ArrayData = vortex_array::arrays::decimal::DecimalData -pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Decimal::OperationsVTable = vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::Decimal @@ -21110,25 +20228,19 @@ pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Decimal::dtype(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Decimal::len(array: &vortex_array::arrays::decimal::DecimalData) -> usize +pub fn vortex_array::arrays::Decimal::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Decimal::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21138,15 +20250,13 @@ pub fn vortex_array::arrays::Decimal::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21154,8 +20264,6 @@ impl vortex_array::VTable for vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData -pub type vortex_array::arrays::Extension::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -21170,25 +20278,19 @@ pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Extension::dtype(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Extension::len(array: &vortex_array::arrays::extension::ExtensionData) -> usize +pub fn vortex_array::arrays::Extension::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Extension::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21198,15 +20300,13 @@ pub fn vortex_array::arrays::Extension::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Extension::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Extension::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21214,8 +20314,6 @@ impl vortex_array::VTable for vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ArrayData = vortex_array::arrays::filter::FilterData -pub type vortex_array::arrays::Filter::Metadata = vortex_array::arrays::filter::vtable::FilterMetadata - pub type vortex_array::arrays::Filter::OperationsVTable = vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Filter @@ -21230,25 +20328,19 @@ pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::filter::vtable::FilterMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Filter::dtype(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Filter::len(array: &vortex_array::arrays::filter::FilterData) -> usize +pub fn vortex_array::arrays::Filter::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Filter::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21258,15 +20350,13 @@ pub fn vortex_array::arrays::Filter::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Filter::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Filter::vtable(_array: &vortex_array::arrays::filter::FilterData) -> &Self +pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21274,8 +20364,6 @@ impl vortex_array::VTable for vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ArrayData = vortex_array::arrays::fixed_size_list::FixedSizeListData -pub type vortex_array::arrays::FixedSizeList::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::FixedSizeList::OperationsVTable = vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arrays::FixedSizeList @@ -21290,25 +20378,19 @@ pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::FixedSizeList::dtype(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::FixedSizeList::len(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> usize +pub fn vortex_array::arrays::FixedSizeList::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::FixedSizeList::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21318,15 +20400,13 @@ pub fn vortex_array::arrays::FixedSizeList::reduce(array: vortex_array::ArrayVie pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::FixedSizeList::vtable(_array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &Self +pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21334,8 +20414,6 @@ impl vortex_array::VTable for vortex_array::arrays::List pub type vortex_array::arrays::List::ArrayData = vortex_array::arrays::list::ListData -pub type vortex_array::arrays::List::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::List::OperationsVTable = vortex_array::arrays::List pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List @@ -21350,25 +20428,19 @@ pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::List::dtype(array: &vortex_array::arrays::list::ListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::List::len(array: &vortex_array::arrays::list::ListData) -> usize +pub fn vortex_array::arrays::List::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::List::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21378,15 +20450,13 @@ pub fn vortex_array::arrays::List::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::List::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::list::ListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::List::vtable(_array: &vortex_array::arrays::list::ListData) -> &Self +pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21394,8 +20464,6 @@ impl vortex_array::VTable for vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ArrayData = vortex_array::arrays::listview::ListViewData -pub type vortex_array::arrays::ListView::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::ListView::OperationsVTable = vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays::ListView @@ -21410,25 +20478,19 @@ pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::ListView::dtype(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::ListView::len(array: &vortex_array::arrays::listview::ListViewData) -> usize +pub fn vortex_array::arrays::ListView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::ListView::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21438,15 +20500,13 @@ pub fn vortex_array::arrays::ListView::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::ListView::vtable(_array: &vortex_array::arrays::listview::ListViewData) -> &Self +pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21454,8 +20514,6 @@ impl vortex_array::VTable for vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ArrayData = vortex_array::arrays::masked::MaskedData -pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked @@ -21470,25 +20528,19 @@ pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Masked::dtype(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Masked::len(array: &vortex_array::arrays::masked::MaskedData) -> usize +pub fn vortex_array::arrays::Masked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Masked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21498,15 +20550,13 @@ pub fn vortex_array::arrays::Masked::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Masked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Masked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21514,8 +20564,6 @@ impl vortex_array::VTable for vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ArrayData = vortex_array::arrays::primitive::PrimitiveData -pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive @@ -21530,25 +20578,19 @@ pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Primitive::dtype(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Primitive::len(array: &vortex_array::arrays::primitive::PrimitiveData) -> usize +pub fn vortex_array::arrays::Primitive::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Primitive::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21558,15 +20600,13 @@ pub fn vortex_array::arrays::Primitive::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21574,8 +20614,6 @@ impl vortex_array::VTable for vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ArrayData = vortex_array::arrays::shared::SharedData -pub type vortex_array::arrays::Shared::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Shared::OperationsVTable = vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Shared @@ -21590,25 +20628,19 @@ pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Shared::dtype(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Shared::len(array: &vortex_array::arrays::shared::SharedData) -> usize +pub fn vortex_array::arrays::Shared::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Shared::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21618,15 +20650,13 @@ pub fn vortex_array::arrays::Shared::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Shared::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Shared::vtable(_array: &vortex_array::arrays::shared::SharedData) -> &Self +pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21634,8 +20664,6 @@ impl vortex_array::VTable for vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ArrayData = vortex_array::arrays::struct_::StructData -pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct @@ -21650,25 +20678,19 @@ pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Struct::dtype(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Struct::len(array: &vortex_array::arrays::struct_::StructData) -> usize +pub fn vortex_array::arrays::Struct::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Struct::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21678,15 +20700,13 @@ pub fn vortex_array::arrays::Struct::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Struct::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Struct::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21694,8 +20714,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ArrayData = vortex_array::arrays::varbin::VarBinData -pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::VarBin::OperationsVTable = vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::VarBin @@ -21710,25 +20728,19 @@ pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBin::dtype(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::VarBin::len(array: &vortex_array::arrays::varbin::VarBinData) -> usize +pub fn vortex_array::arrays::VarBin::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::VarBin::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21738,15 +20750,13 @@ pub fn vortex_array::arrays::VarBin::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBin::vtable(_array: &vortex_array::arrays::varbin::VarBinData) -> &Self +pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21754,8 +20764,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ArrayData = vortex_array::arrays::varbinview::VarBinViewData -pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView @@ -21770,25 +20778,19 @@ pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBinView::dtype(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::VarBinView::len(array: &vortex_array::arrays::varbinview::VarBinViewData) -> usize +pub fn vortex_array::arrays::VarBinView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::VarBinView::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21798,15 +20800,13 @@ pub fn vortex_array::arrays::VarBinView::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBinView::vtable(_array: &vortex_array::arrays::varbinview::VarBinViewData) -> &Self +pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21814,8 +20814,6 @@ impl vortex_array::VTable for vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ArrayData = vortex_array::arrays::variant::VariantData -pub type vortex_array::arrays::Variant::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Variant::OperationsVTable = vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::Variant @@ -21830,25 +20828,19 @@ pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Variant::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Variant::len(array: &Self::ArrayData) -> usize +pub fn vortex_array::arrays::Variant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Variant::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21858,15 +20850,13 @@ pub fn vortex_array::arrays::Variant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Variant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21874,8 +20864,6 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData -pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -21890,25 +20878,19 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize +pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21918,15 +20900,13 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21934,8 +20914,6 @@ impl vortex_array::VTable for vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ArrayData = vortex_array::arrays::null::NullData -pub type vortex_array::arrays::null::Null::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::null::Null::OperationsVTable = vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays::null::Null @@ -21950,25 +20928,19 @@ pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::null::Null::dtype(_array: &vortex_array::arrays::null::NullData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::null::Null::len(array: &vortex_array::arrays::null::NullData) -> usize +pub fn vortex_array::arrays::null::Null::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::null::Null::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21978,15 +20950,13 @@ pub fn vortex_array::arrays::null::Null::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::null::Null::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::null::Null::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21994,8 +20964,6 @@ impl vortex_array::VTable for vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray -pub type vortex_array::arrays::patched::Patched::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -22010,25 +20978,19 @@ pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::patched::Patched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::patched::Patched::len(array: &Self::ArrayData) -> usize +pub fn vortex_array::arrays::patched::Patched::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::patched::Patched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22038,15 +21000,13 @@ pub fn vortex_array::arrays::patched::Patched::reduce(array: vortex_array::Array pub fn vortex_array::arrays::patched::Patched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::patched::Patched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::patched::Patched::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -22054,8 +21014,6 @@ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::Metadata = vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata - pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -22070,25 +21028,19 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::dtype(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::len(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> usize - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22098,15 +21050,13 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_arr pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::vtable(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &Self +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -22114,8 +21064,6 @@ impl vortex_array::VTable for vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ArrayData = vortex_array::arrays::slice::SliceData -pub type vortex_array::arrays::slice::Slice::Metadata = vortex_array::arrays::slice::SliceMetadata - pub type vortex_array::arrays::slice::Slice::OperationsVTable = vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arrays::slice::Slice @@ -22130,25 +21078,19 @@ pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::slice::SliceMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::slice::Slice::len(array: &vortex_array::arrays::slice::SliceData) -> usize +pub fn vortex_array::arrays::slice::Slice::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::slice::Slice::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22158,15 +21100,13 @@ pub fn vortex_array::arrays::slice::Slice::reduce(array: vortex_array::ArrayView pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::slice::Slice::vtable(_array: &vortex_array::arrays::slice::SliceData) -> &Self +pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -22562,8 +21502,6 @@ pub fn vortex_array::Array::from_indices::into_bit_buffer(self) -> vortex_buffer::bit::buf::BitBuffer -pub fn vortex_array::Array::into_parts(self) -> vortex_array::arrays::bool::BoolArrayParts - pub fn vortex_array::Array::new(bits: vortex_buffer::bit::buf::BitBuffer, validity: vortex_array::validity::Validity) -> Self pub fn vortex_array::Array::new_handle(handle: vortex_array::buffer::BufferHandle, offset: usize, len: usize, validity: vortex_array::validity::Validity) -> Self @@ -22584,6 +21522,8 @@ impl vortex_array::Array pub unsafe fn vortex_array::Array::new_unchecked(chunks: alloc::vec::Vec, dtype: vortex_array::dtype::DType) -> Self +pub fn vortex_array::Array::rechunk(&self, target_bytesize: u64, target_rowsize: usize) -> vortex_error::VortexResult + pub fn vortex_array::Array::try_new(chunks: alloc::vec::Vec, dtype: vortex_array::dtype::DType) -> vortex_error::VortexResult impl vortex_array::Array @@ -22688,6 +21628,8 @@ impl vortex_array::Array pub fn vortex_array::Array::narrow(&self) -> vortex_error::VortexResult +pub fn vortex_array::Array::reinterpret_cast(&self, ptype: vortex_array::dtype::PType) -> Self + impl vortex_array::Array pub fn vortex_array::Array::patch(self, patches: &vortex_array::patches::Patches, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -22704,14 +21646,14 @@ impl vortex_array::Array pub fn vortex_array::Array::from_fields>(items: &[(N, vortex_array::ArrayRef)]) -> vortex_error::VortexResult -pub fn vortex_array::Array::into_parts(self) -> vortex_array::arrays::struct_::StructArrayParts - pub fn vortex_array::Array::new(names: vortex_array::dtype::FieldNames, fields: impl core::convert::Into>, length: usize, validity: vortex_array::validity::Validity) -> Self pub fn vortex_array::Array::new_fieldless_with_len(len: usize) -> Self pub unsafe fn vortex_array::Array::new_unchecked(fields: impl core::convert::Into>, dtype: vortex_array::dtype::StructFields, length: usize, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_array::Array::project(&self, projection: &[vortex_array::dtype::FieldName]) -> vortex_error::VortexResult + pub fn vortex_array::Array::try_from_iter, A: vortex_array::IntoArray, T: core::iter::traits::collect::IntoIterator>(iter: T) -> vortex_error::VortexResult pub fn vortex_array::Array::try_from_iter_with_validity, A: vortex_array::IntoArray, T: core::iter::traits::collect::IntoIterator>(iter: T, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult @@ -22860,6 +21802,8 @@ pub fn vortex_array::Array::encoding_id(&self) -> vortex_array::ArrayId pub fn vortex_array::Array::into_data(self) -> ::ArrayData +pub fn vortex_array::Array::into_parts(self) -> vortex_array::ArrayParts + pub fn vortex_array::Array::is_empty(&self) -> bool pub fn vortex_array::Array::len(&self) -> usize @@ -22868,7 +21812,9 @@ pub fn vortex_array::Array::statistics(&self) -> vortex_array::stats::StatsSe pub fn vortex_array::Array::try_from_array_ref(array: vortex_array::ArrayRef) -> core::result::Result -pub fn vortex_array::Array::try_from_data(data: ::ArrayData) -> vortex_error::VortexResult +pub fn vortex_array::Array::try_from_parts(new: vortex_array::ArrayParts) -> vortex_error::VortexResult + +pub fn vortex_array::Array::with_stats_set(self, stats: vortex_array::stats::StatsSet) -> Self impl core::convert::From> for vortex_array::Array @@ -22970,6 +21916,20 @@ impl vortex_array::IntoArray for vortex_array::Array pub fn vortex_array::Array::into_array(self) -> vortex_array::ArrayRef +pub struct vortex_array::ArrayParts + +pub vortex_array::ArrayParts::data: ::ArrayData + +pub vortex_array::ArrayParts::dtype: vortex_array::dtype::DType + +pub vortex_array::ArrayParts::len: usize + +pub vortex_array::ArrayParts::vtable: V + +impl vortex_array::ArrayParts + +pub fn vortex_array::ArrayParts::new(vtable: V, dtype: vortex_array::dtype::DType, len: usize, data: ::ArrayData) -> Self + pub struct vortex_array::ArrayRef(_) impl vortex_array::ArrayRef @@ -23146,93 +22106,13 @@ impl core::convert::From for vortex_array::ArrayRef pub fn vortex_array::ArrayRef::from(value: vortex_array::Canonical) -> Self -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::bool::BoolData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::chunked::ChunkedData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::constant::ConstantData) -> vortex_array::ArrayRef - impl core::convert::From for vortex_array::ArrayRef pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::datetime::TemporalData) -> Self -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::decimal::DecimalData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::dict::DictData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::extension::ExtensionData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::filter::FilterData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::fixed_size_list::FixedSizeListData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::list::ListData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::listview::ListViewData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::masked::MaskedData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::null::NullData) -> vortex_array::ArrayRef - impl core::convert::From for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::primitive::PrimitiveData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::scalar_fn::ScalarFnData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::shared::SharedData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::slice::SliceData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::struct_::StructData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::varbin::VarBinData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::varbinview::VarBinViewData) -> vortex_array::ArrayRef - -impl core::convert::From for vortex_array::ArrayRef - -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::variant::VariantData) -> vortex_array::ArrayRef +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> Self impl core::convert::TryFrom<&vortex_array::ArrayRef> for arrow_array::record_batch::RecordBatch @@ -23814,9 +22694,7 @@ pub fn core::option::Option::array_hash(&self, state: pub trait vortex_array::ArrayVTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug -pub type vortex_array::ArrayVTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug + vortex_array::IntoArray - -pub type vortex_array::ArrayVTable::Metadata: core::fmt::Debug +pub type vortex_array::ArrayVTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug pub type vortex_array::ArrayVTable::OperationsVTable: vortex_array::OperationsVTable @@ -23832,25 +22710,19 @@ pub fn vortex_array::ArrayVTable::buffer(array: vortex_array::ArrayView<'_, Self pub fn vortex_array::ArrayVTable::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::ArrayVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::ArrayVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::ArrayVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::ArrayVTable::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::ArrayVTable::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::ArrayVTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::ArrayVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::ArrayVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::ArrayVTable::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::ArrayVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::ArrayVTable::len(array: &Self::ArrayData) -> usize - -pub fn vortex_array::ArrayVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::ArrayVTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::ArrayVTable::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23860,15 +22732,13 @@ pub fn vortex_array::ArrayVTable::reduce(array: vortex_array::ArrayView<'_, Self pub fn vortex_array::ArrayVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::ArrayVTable::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::ArrayVTable::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::ArrayVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::ArrayVTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] -pub fn vortex_array::ArrayVTable::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::ArrayVTable::vtable(array: &Self::ArrayData) -> &Self +pub fn vortex_array::ArrayVTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::ArrayVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23876,8 +22746,6 @@ impl vortex_array::VTable for vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ArrayData = vortex_array::arrays::bool::BoolData -pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Bool::OperationsVTable = vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool @@ -23892,25 +22760,19 @@ pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Bool::dtype(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Bool::len(array: &vortex_array::arrays::bool::BoolData) -> usize +pub fn vortex_array::arrays::Bool::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Bool::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23920,15 +22782,13 @@ pub fn vortex_array::arrays::Bool::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Bool::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23936,8 +22796,6 @@ impl vortex_array::VTable for vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ArrayData = vortex_array::arrays::chunked::ChunkedData -pub type vortex_array::arrays::Chunked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Chunked::OperationsVTable = vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::Chunked @@ -23952,25 +22810,19 @@ pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Chunked::dtype(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Chunked::len(array: &vortex_array::arrays::chunked::ChunkedData) -> usize +pub fn vortex_array::arrays::Chunked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Chunked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23980,15 +22832,13 @@ pub fn vortex_array::arrays::Chunked::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Chunked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23996,8 +22846,6 @@ impl vortex_array::VTable for vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ArrayData = vortex_array::arrays::constant::ConstantData -pub type vortex_array::arrays::Constant::Metadata = vortex_array::scalar::Scalar - pub type vortex_array::arrays::Constant::OperationsVTable = vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays::Constant @@ -24012,25 +22860,19 @@ pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Constant::dtype(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Constant::len(array: &vortex_array::arrays::constant::ConstantData) -> usize +pub fn vortex_array::arrays::Constant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Constant::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24040,15 +22882,13 @@ pub fn vortex_array::arrays::Constant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Constant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24056,8 +22896,6 @@ impl vortex_array::VTable for vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ArrayData = vortex_array::arrays::decimal::DecimalData -pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Decimal::OperationsVTable = vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::Decimal @@ -24072,25 +22910,19 @@ pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Decimal::dtype(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Decimal::len(array: &vortex_array::arrays::decimal::DecimalData) -> usize +pub fn vortex_array::arrays::Decimal::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Decimal::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24100,15 +22932,13 @@ pub fn vortex_array::arrays::Decimal::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24116,8 +22946,6 @@ impl vortex_array::VTable for vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData -pub type vortex_array::arrays::Extension::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -24132,25 +22960,19 @@ pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Extension::dtype(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Extension::len(array: &vortex_array::arrays::extension::ExtensionData) -> usize +pub fn vortex_array::arrays::Extension::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Extension::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24160,15 +22982,13 @@ pub fn vortex_array::arrays::Extension::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Extension::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Extension::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24176,8 +22996,6 @@ impl vortex_array::VTable for vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ArrayData = vortex_array::arrays::filter::FilterData -pub type vortex_array::arrays::Filter::Metadata = vortex_array::arrays::filter::vtable::FilterMetadata - pub type vortex_array::arrays::Filter::OperationsVTable = vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Filter @@ -24192,25 +23010,19 @@ pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::filter::vtable::FilterMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Filter::dtype(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Filter::len(array: &vortex_array::arrays::filter::FilterData) -> usize +pub fn vortex_array::arrays::Filter::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Filter::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24220,15 +23032,13 @@ pub fn vortex_array::arrays::Filter::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Filter::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Filter::vtable(_array: &vortex_array::arrays::filter::FilterData) -> &Self +pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24236,8 +23046,6 @@ impl vortex_array::VTable for vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ArrayData = vortex_array::arrays::fixed_size_list::FixedSizeListData -pub type vortex_array::arrays::FixedSizeList::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::FixedSizeList::OperationsVTable = vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arrays::FixedSizeList @@ -24252,25 +23060,19 @@ pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::FixedSizeList::dtype(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::FixedSizeList::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::FixedSizeList::len(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> usize - -pub fn vortex_array::arrays::FixedSizeList::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24280,15 +23082,13 @@ pub fn vortex_array::arrays::FixedSizeList::reduce(array: vortex_array::ArrayVie pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::FixedSizeList::vtable(_array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &Self +pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24296,8 +23096,6 @@ impl vortex_array::VTable for vortex_array::arrays::List pub type vortex_array::arrays::List::ArrayData = vortex_array::arrays::list::ListData -pub type vortex_array::arrays::List::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::List::OperationsVTable = vortex_array::arrays::List pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List @@ -24312,25 +23110,19 @@ pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::List::dtype(array: &vortex_array::arrays::list::ListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::List::len(array: &vortex_array::arrays::list::ListData) -> usize +pub fn vortex_array::arrays::List::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::List::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24340,15 +23132,13 @@ pub fn vortex_array::arrays::List::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::List::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::list::ListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::List::vtable(_array: &vortex_array::arrays::list::ListData) -> &Self +pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24356,8 +23146,6 @@ impl vortex_array::VTable for vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ArrayData = vortex_array::arrays::listview::ListViewData -pub type vortex_array::arrays::ListView::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::ListView::OperationsVTable = vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays::ListView @@ -24372,25 +23160,19 @@ pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::ListView::dtype(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::ListView::len(array: &vortex_array::arrays::listview::ListViewData) -> usize +pub fn vortex_array::arrays::ListView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::ListView::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24400,15 +23182,13 @@ pub fn vortex_array::arrays::ListView::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::ListView::vtable(_array: &vortex_array::arrays::listview::ListViewData) -> &Self +pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24416,8 +23196,6 @@ impl vortex_array::VTable for vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ArrayData = vortex_array::arrays::masked::MaskedData -pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked @@ -24432,25 +23210,19 @@ pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Masked::dtype(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Masked::len(array: &vortex_array::arrays::masked::MaskedData) -> usize +pub fn vortex_array::arrays::Masked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Masked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24460,15 +23232,13 @@ pub fn vortex_array::arrays::Masked::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Masked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Masked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24476,8 +23246,6 @@ impl vortex_array::VTable for vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ArrayData = vortex_array::arrays::primitive::PrimitiveData -pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive @@ -24492,25 +23260,19 @@ pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Primitive::dtype(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Primitive::len(array: &vortex_array::arrays::primitive::PrimitiveData) -> usize +pub fn vortex_array::arrays::Primitive::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Primitive::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24520,15 +23282,13 @@ pub fn vortex_array::arrays::Primitive::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24536,8 +23296,6 @@ impl vortex_array::VTable for vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ArrayData = vortex_array::arrays::shared::SharedData -pub type vortex_array::arrays::Shared::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Shared::OperationsVTable = vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Shared @@ -24552,25 +23310,19 @@ pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Shared::dtype(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Shared::len(array: &vortex_array::arrays::shared::SharedData) -> usize +pub fn vortex_array::arrays::Shared::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Shared::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24580,15 +23332,13 @@ pub fn vortex_array::arrays::Shared::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Shared::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Shared::vtable(_array: &vortex_array::arrays::shared::SharedData) -> &Self +pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24596,8 +23346,6 @@ impl vortex_array::VTable for vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ArrayData = vortex_array::arrays::struct_::StructData -pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct @@ -24612,25 +23360,19 @@ pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Struct::dtype(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Struct::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Struct::len(array: &vortex_array::arrays::struct_::StructData) -> usize - -pub fn vortex_array::arrays::Struct::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24640,15 +23382,13 @@ pub fn vortex_array::arrays::Struct::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Struct::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Struct::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24656,8 +23396,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ArrayData = vortex_array::arrays::varbin::VarBinData -pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::VarBin::OperationsVTable = vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::VarBin @@ -24672,25 +23410,19 @@ pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBin::dtype(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::VarBin::len(array: &vortex_array::arrays::varbin::VarBinData) -> usize +pub fn vortex_array::arrays::VarBin::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::VarBin::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24700,15 +23432,13 @@ pub fn vortex_array::arrays::VarBin::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBin::vtable(_array: &vortex_array::arrays::varbin::VarBinData) -> &Self +pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24716,8 +23446,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ArrayData = vortex_array::arrays::varbinview::VarBinViewData -pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView @@ -24732,25 +23460,19 @@ pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBinView::dtype(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::VarBinView::len(array: &vortex_array::arrays::varbinview::VarBinViewData) -> usize +pub fn vortex_array::arrays::VarBinView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::VarBinView::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24760,15 +23482,13 @@ pub fn vortex_array::arrays::VarBinView::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBinView::vtable(_array: &vortex_array::arrays::varbinview::VarBinViewData) -> &Self +pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24776,8 +23496,6 @@ impl vortex_array::VTable for vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ArrayData = vortex_array::arrays::variant::VariantData -pub type vortex_array::arrays::Variant::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Variant::OperationsVTable = vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::Variant @@ -24792,25 +23510,19 @@ pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Variant::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Variant::len(array: &Self::ArrayData) -> usize +pub fn vortex_array::arrays::Variant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Variant::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24820,15 +23532,13 @@ pub fn vortex_array::arrays::Variant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Variant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24836,8 +23546,6 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData -pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -24852,25 +23560,19 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize +pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24880,15 +23582,13 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24896,8 +23596,6 @@ impl vortex_array::VTable for vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ArrayData = vortex_array::arrays::null::NullData -pub type vortex_array::arrays::null::Null::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::null::Null::OperationsVTable = vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays::null::Null @@ -24912,25 +23610,19 @@ pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::null::Null::dtype(_array: &vortex_array::arrays::null::NullData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::null::Null::len(array: &vortex_array::arrays::null::NullData) -> usize +pub fn vortex_array::arrays::null::Null::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::null::Null::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24940,15 +23632,13 @@ pub fn vortex_array::arrays::null::Null::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::null::Null::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::null::Null::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24956,8 +23646,6 @@ impl vortex_array::VTable for vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray -pub type vortex_array::arrays::patched::Patched::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -24972,25 +23660,19 @@ pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::patched::Patched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::patched::Patched::len(array: &Self::ArrayData) -> usize +pub fn vortex_array::arrays::patched::Patched::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::patched::Patched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25000,15 +23682,13 @@ pub fn vortex_array::arrays::patched::Patched::reduce(array: vortex_array::Array pub fn vortex_array::arrays::patched::Patched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::patched::Patched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::patched::Patched::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25016,8 +23696,6 @@ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::Metadata = vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata - pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -25032,25 +23710,19 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::dtype(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::len(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> usize - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25060,15 +23732,13 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_arr pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::vtable(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &Self +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25076,8 +23746,6 @@ impl vortex_array::VTable for vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ArrayData = vortex_array::arrays::slice::SliceData -pub type vortex_array::arrays::slice::Slice::Metadata = vortex_array::arrays::slice::SliceMetadata - pub type vortex_array::arrays::slice::Slice::OperationsVTable = vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arrays::slice::Slice @@ -25092,25 +23760,19 @@ pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::slice::SliceMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::slice::Slice::len(array: &vortex_array::arrays::slice::SliceData) -> usize +pub fn vortex_array::arrays::slice::Slice::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::slice::Slice::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25120,15 +23782,13 @@ pub fn vortex_array::arrays::slice::Slice::reduce(array: vortex_array::ArrayView pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::slice::Slice::vtable(_array: &vortex_array::arrays::slice::SliceData) -> &Self +pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25304,94 +23964,14 @@ impl vortex_array::IntoArray for vortex_array::Columnar pub fn vortex_array::Columnar::into_array(self) -> vortex_array::ArrayRef -impl vortex_array::IntoArray for vortex_array::arrays::bool::BoolData - -pub fn vortex_array::arrays::bool::BoolData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::chunked::ChunkedData - -pub fn vortex_array::arrays::chunked::ChunkedData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::constant::ConstantData - -pub fn vortex_array::arrays::constant::ConstantData::into_array(self) -> vortex_array::ArrayRef - impl vortex_array::IntoArray for vortex_array::arrays::datetime::TemporalData pub fn vortex_array::arrays::datetime::TemporalData::into_array(self) -> vortex_array::ArrayRef -impl vortex_array::IntoArray for vortex_array::arrays::decimal::DecimalData - -pub fn vortex_array::arrays::decimal::DecimalData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::dict::DictData - -pub fn vortex_array::arrays::dict::DictData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::filter::FilterData - -pub fn vortex_array::arrays::filter::FilterData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::fixed_size_list::FixedSizeListData - -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::list::ListData - -pub fn vortex_array::arrays::list::ListData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::listview::ListViewData - -pub fn vortex_array::arrays::listview::ListViewData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::masked::MaskedData - -pub fn vortex_array::arrays::masked::MaskedData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::null::NullData - -pub fn vortex_array::arrays::null::NullData::into_array(self) -> vortex_array::ArrayRef - impl vortex_array::IntoArray for vortex_array::arrays::patched::PatchedArray pub fn vortex_array::arrays::patched::PatchedArray::into_array(self) -> vortex_array::ArrayRef -impl vortex_array::IntoArray for vortex_array::arrays::primitive::PrimitiveData - -pub fn vortex_array::arrays::primitive::PrimitiveData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::scalar_fn::ScalarFnData - -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::shared::SharedData - -pub fn vortex_array::arrays::shared::SharedData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::slice::SliceData - -pub fn vortex_array::arrays::slice::SliceData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::struct_::StructData - -pub fn vortex_array::arrays::struct_::StructData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::varbin::VarBinData - -pub fn vortex_array::arrays::varbin::VarBinData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::varbinview::VarBinViewData - -pub fn vortex_array::arrays::varbinview::VarBinViewData::into_array(self) -> vortex_array::ArrayRef - -impl vortex_array::IntoArray for vortex_array::arrays::variant::VariantData - -pub fn vortex_array::arrays::variant::VariantData::into_array(self) -> vortex_array::ArrayRef - impl vortex_array::IntoArray for vortex_buffer::bit::buf::BitBuffer pub fn vortex_buffer::bit::buf::BitBuffer::into_array(self) -> vortex_array::ArrayRef @@ -25578,9 +24158,7 @@ pub fn vortex_array::ArrayRef::to_varbinview(&self) -> vortex_array::arrays::Var pub trait vortex_array::VTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug -pub type vortex_array::VTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug + vortex_array::IntoArray - -pub type vortex_array::VTable::Metadata: core::fmt::Debug +pub type vortex_array::VTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug pub type vortex_array::VTable::OperationsVTable: vortex_array::OperationsVTable @@ -25596,25 +24174,19 @@ pub fn vortex_array::VTable::buffer(array: vortex_array::ArrayView<'_, Self>, id pub fn vortex_array::VTable::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::VTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::VTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::VTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::VTable::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::VTable::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::VTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::VTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::VTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::VTable::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::VTable::len(array: &Self::ArrayData) -> usize +pub fn vortex_array::VTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::VTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::VTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::VTable::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25624,15 +24196,13 @@ pub fn vortex_array::VTable::reduce(array: vortex_array::ArrayView<'_, Self>) -> pub fn vortex_array::VTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::VTable::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::VTable::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::VTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::VTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] -pub fn vortex_array::VTable::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::VTable::vtable(array: &Self::ArrayData) -> &Self +pub fn vortex_array::VTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::VTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25640,8 +24210,6 @@ impl vortex_array::VTable for vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ArrayData = vortex_array::arrays::bool::BoolData -pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Bool::OperationsVTable = vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool @@ -25656,25 +24224,19 @@ pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Bool::dtype(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Bool::len(array: &vortex_array::arrays::bool::BoolData) -> usize +pub fn vortex_array::arrays::Bool::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Bool::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25684,15 +24246,13 @@ pub fn vortex_array::arrays::Bool::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Bool::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25700,8 +24260,6 @@ impl vortex_array::VTable for vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ArrayData = vortex_array::arrays::chunked::ChunkedData -pub type vortex_array::arrays::Chunked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Chunked::OperationsVTable = vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::Chunked @@ -25716,25 +24274,19 @@ pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Chunked::dtype(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Chunked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Chunked::len(array: &vortex_array::arrays::chunked::ChunkedData) -> usize - -pub fn vortex_array::arrays::Chunked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25744,15 +24296,13 @@ pub fn vortex_array::arrays::Chunked::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Chunked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25760,8 +24310,6 @@ impl vortex_array::VTable for vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ArrayData = vortex_array::arrays::constant::ConstantData -pub type vortex_array::arrays::Constant::Metadata = vortex_array::scalar::Scalar - pub type vortex_array::arrays::Constant::OperationsVTable = vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays::Constant @@ -25776,25 +24324,19 @@ pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Constant::dtype(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Constant::len(array: &vortex_array::arrays::constant::ConstantData) -> usize +pub fn vortex_array::arrays::Constant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Constant::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25804,15 +24346,13 @@ pub fn vortex_array::arrays::Constant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Constant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25820,8 +24360,6 @@ impl vortex_array::VTable for vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ArrayData = vortex_array::arrays::decimal::DecimalData -pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::Decimal::OperationsVTable = vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::Decimal @@ -25836,25 +24374,19 @@ pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Decimal::dtype(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Decimal::len(array: &vortex_array::arrays::decimal::DecimalData) -> usize +pub fn vortex_array::arrays::Decimal::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Decimal::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25864,15 +24396,13 @@ pub fn vortex_array::arrays::Decimal::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25880,8 +24410,6 @@ impl vortex_array::VTable for vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData -pub type vortex_array::arrays::Extension::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -25896,25 +24424,19 @@ pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Extension::dtype(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Extension::len(array: &vortex_array::arrays::extension::ExtensionData) -> usize +pub fn vortex_array::arrays::Extension::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Extension::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25924,15 +24446,13 @@ pub fn vortex_array::arrays::Extension::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Extension::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Extension::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25940,8 +24460,6 @@ impl vortex_array::VTable for vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ArrayData = vortex_array::arrays::filter::FilterData -pub type vortex_array::arrays::Filter::Metadata = vortex_array::arrays::filter::vtable::FilterMetadata - pub type vortex_array::arrays::Filter::OperationsVTable = vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Filter @@ -25956,25 +24474,19 @@ pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::filter::vtable::FilterMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Filter::dtype(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Filter::len(array: &vortex_array::arrays::filter::FilterData) -> usize +pub fn vortex_array::arrays::Filter::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Filter::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25984,15 +24496,13 @@ pub fn vortex_array::arrays::Filter::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Filter::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Filter::vtable(_array: &vortex_array::arrays::filter::FilterData) -> &Self +pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26000,8 +24510,6 @@ impl vortex_array::VTable for vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ArrayData = vortex_array::arrays::fixed_size_list::FixedSizeListData -pub type vortex_array::arrays::FixedSizeList::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::FixedSizeList::OperationsVTable = vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arrays::FixedSizeList @@ -26016,25 +24524,19 @@ pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::FixedSizeList::dtype(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::FixedSizeList::len(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> usize +pub fn vortex_array::arrays::FixedSizeList::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::FixedSizeList::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26044,15 +24546,13 @@ pub fn vortex_array::arrays::FixedSizeList::reduce(array: vortex_array::ArrayVie pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::FixedSizeList::vtable(_array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &Self +pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26060,8 +24560,6 @@ impl vortex_array::VTable for vortex_array::arrays::List pub type vortex_array::arrays::List::ArrayData = vortex_array::arrays::list::ListData -pub type vortex_array::arrays::List::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::List::OperationsVTable = vortex_array::arrays::List pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List @@ -26076,25 +24574,19 @@ pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::List::dtype(array: &vortex_array::arrays::list::ListData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::List::len(array: &vortex_array::arrays::list::ListData) -> usize +pub fn vortex_array::arrays::List::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::List::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26104,15 +24596,13 @@ pub fn vortex_array::arrays::List::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::List::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::list::ListData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::List::vtable(_array: &vortex_array::arrays::list::ListData) -> &Self +pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26120,8 +24610,6 @@ impl vortex_array::VTable for vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ArrayData = vortex_array::arrays::listview::ListViewData -pub type vortex_array::arrays::ListView::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::ListView::OperationsVTable = vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays::ListView @@ -26136,25 +24624,19 @@ pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::ListView::dtype(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::ListView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::ListView::len(array: &vortex_array::arrays::listview::ListViewData) -> usize - -pub fn vortex_array::arrays::ListView::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26164,15 +24646,13 @@ pub fn vortex_array::arrays::ListView::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::ListView::vtable(_array: &vortex_array::arrays::listview::ListViewData) -> &Self +pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26180,8 +24660,6 @@ impl vortex_array::VTable for vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ArrayData = vortex_array::arrays::masked::MaskedData -pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked @@ -26196,25 +24674,19 @@ pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Masked::dtype(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Masked::len(array: &vortex_array::arrays::masked::MaskedData) -> usize +pub fn vortex_array::arrays::Masked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Masked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26224,15 +24696,13 @@ pub fn vortex_array::arrays::Masked::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Masked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Masked::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26240,8 +24710,6 @@ impl vortex_array::VTable for vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ArrayData = vortex_array::arrays::primitive::PrimitiveData -pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive @@ -26256,25 +24724,19 @@ pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Primitive::dtype(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Primitive::len(array: &vortex_array::arrays::primitive::PrimitiveData) -> usize +pub fn vortex_array::arrays::Primitive::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Primitive::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26284,15 +24746,13 @@ pub fn vortex_array::arrays::Primitive::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26300,8 +24760,6 @@ impl vortex_array::VTable for vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ArrayData = vortex_array::arrays::shared::SharedData -pub type vortex_array::arrays::Shared::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Shared::OperationsVTable = vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Shared @@ -26316,25 +24774,19 @@ pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Shared::dtype(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Shared::len(array: &vortex_array::arrays::shared::SharedData) -> usize +pub fn vortex_array::arrays::Shared::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Shared::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26344,15 +24796,13 @@ pub fn vortex_array::arrays::Shared::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Shared::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Shared::vtable(_array: &vortex_array::arrays::shared::SharedData) -> &Self +pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26360,8 +24810,6 @@ impl vortex_array::VTable for vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ArrayData = vortex_array::arrays::struct_::StructData -pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct @@ -26376,25 +24824,19 @@ pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Struct::dtype(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Struct::len(array: &vortex_array::arrays::struct_::StructData) -> usize +pub fn vortex_array::arrays::Struct::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Struct::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26404,15 +24846,13 @@ pub fn vortex_array::arrays::Struct::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Struct::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Struct::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26420,8 +24860,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ArrayData = vortex_array::arrays::varbin::VarBinData -pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::VarBin::OperationsVTable = vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::VarBin @@ -26436,25 +24874,19 @@ pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBin::dtype(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::VarBin::len(array: &vortex_array::arrays::varbin::VarBinData) -> usize +pub fn vortex_array::arrays::VarBin::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::VarBin::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26464,15 +24896,13 @@ pub fn vortex_array::arrays::VarBin::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBin::vtable(_array: &vortex_array::arrays::varbin::VarBinData) -> &Self +pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26480,8 +24910,6 @@ impl vortex_array::VTable for vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ArrayData = vortex_array::arrays::varbinview::VarBinViewData -pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView @@ -26496,25 +24924,19 @@ pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::VarBinView::dtype(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::VarBinView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::VarBinView::len(array: &vortex_array::arrays::varbinview::VarBinViewData) -> usize - -pub fn vortex_array::arrays::VarBinView::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26524,15 +24946,13 @@ pub fn vortex_array::arrays::VarBinView::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::VarBinView::vtable(_array: &vortex_array::arrays::varbinview::VarBinViewData) -> &Self +pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26540,8 +24960,6 @@ impl vortex_array::VTable for vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ArrayData = vortex_array::arrays::variant::VariantData -pub type vortex_array::arrays::Variant::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::Variant::OperationsVTable = vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::Variant @@ -26556,25 +24974,19 @@ pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::Variant::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::Variant::len(array: &Self::ArrayData) -> usize +pub fn vortex_array::arrays::Variant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::Variant::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26584,15 +24996,13 @@ pub fn vortex_array::arrays::Variant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::Variant::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26600,8 +25010,6 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData -pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -26616,25 +25024,19 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize +pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26644,15 +25046,13 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26660,8 +25060,6 @@ impl vortex_array::VTable for vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ArrayData = vortex_array::arrays::null::NullData -pub type vortex_array::arrays::null::Null::Metadata = vortex_array::EmptyMetadata - pub type vortex_array::arrays::null::Null::OperationsVTable = vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays::null::Null @@ -26676,25 +25074,19 @@ pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::null::Null::dtype(_array: &vortex_array::arrays::null::NullData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::null::Null::len(array: &vortex_array::arrays::null::NullData) -> usize +pub fn vortex_array::arrays::null::Null::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::null::Null::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26704,15 +25096,13 @@ pub fn vortex_array::arrays::null::Null::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::null::Null::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::null::Null::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26720,8 +25110,6 @@ impl vortex_array::VTable for vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray -pub type vortex_array::arrays::patched::Patched::Metadata = vortex_array::ProstMetadata - pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -26736,25 +25124,19 @@ pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_array::arrays::patched::Patched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::patched::Patched::len(array: &Self::ArrayData) -> usize +pub fn vortex_array::arrays::patched::Patched::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::patched::Patched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26764,15 +25146,13 @@ pub fn vortex_array::arrays::patched::Patched::reduce(array: vortex_array::Array pub fn vortex_array::arrays::patched::Patched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::patched::Patched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::patched::Patched::vtable(_array: &Self::ArrayData) -> &Self +pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26780,8 +25160,6 @@ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::Metadata = vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata - pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -26796,25 +25174,19 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::dtype(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::len(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> usize +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26824,15 +25196,13 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_arr pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::vtable(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &Self +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -26840,8 +25210,6 @@ impl vortex_array::VTable for vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ArrayData = vortex_array::arrays::slice::SliceData -pub type vortex_array::arrays::slice::Slice::Metadata = vortex_array::arrays::slice::SliceMetadata - pub type vortex_array::arrays::slice::Slice::OperationsVTable = vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arrays::slice::Slice @@ -26856,25 +25224,19 @@ pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::slice::SliceMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult - pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId - -pub fn vortex_array::arrays::slice::Slice::len(array: &vortex_array::arrays::slice::SliceData) -> usize +pub fn vortex_array::arrays::slice::Slice::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::slice::Slice::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize @@ -26884,15 +25246,13 @@ pub fn vortex_array::arrays::slice::Slice::reduce(array: vortex_array::ArrayView pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::stats::ArrayStats - -pub fn vortex_array::arrays::slice::Slice::vtable(_array: &vortex_array::arrays::slice::SliceData) -> &Self +pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> diff --git a/vortex-array/src/array/mod.rs b/vortex-array/src/array/mod.rs index 842f759ace7..0dbf065e6a1 100644 --- a/vortex-array/src/array/mod.rs +++ b/vortex-array/src/array/mod.rs @@ -304,15 +304,12 @@ impl DynArray for ArrayInner { fn metadata(&self, this: &ArrayRef) -> VortexResult>> { let view = unsafe { ArrayView::new_unchecked(this, &self.data) }; - V::serialize(V::metadata(view)?) + V::serialize(view) } fn metadata_fmt(&self, this: &ArrayRef, f: &mut Formatter<'_>) -> std::fmt::Result { let view = unsafe { ArrayView::new_unchecked(this, &self.data) }; - match V::metadata(view) { - Err(e) => write!(f, ""), - Ok(metadata) => Debug::fmt(&metadata, f), - } + V::fmt_metadata(view, f) } fn dyn_array_hash(&self, state: &mut dyn Hasher, precision: crate::Precision) { diff --git a/vortex-array/src/array/typed.rs b/vortex-array/src/array/typed.rs index e635a98227d..7918c6b9956 100644 --- a/vortex-array/src/array/typed.rs +++ b/vortex-array/src/array/typed.rs @@ -22,8 +22,28 @@ use crate::array::ArrayView; use crate::array::VTable; use crate::dtype::DType; use crate::stats::ArrayStats; +use crate::stats::StatsSet; use crate::stats::StatsSetRef; use crate::validity::Validity; + +/// Construction parameters for typed arrays. +pub struct ArrayParts { + pub vtable: V, + pub dtype: DType, + pub len: usize, + pub data: V::ArrayData, +} + +impl ArrayParts { + pub fn new(vtable: V, dtype: DType, len: usize, data: V::ArrayData) -> Self { + Self { + vtable, + dtype, + len, + data, + } + } +} // ============================================================================= // ArrayInner — the concrete type stored inside Arc // ============================================================================= @@ -43,14 +63,19 @@ pub(crate) struct ArrayInner { } impl ArrayInner { - /// Create a new inner array from encoding-specific data. + /// Create a new inner array from explicit construction parameters. #[doc(hidden)] - pub fn try_from_data(data: V::ArrayData) -> VortexResult { - let vtable = V::vtable(&data).clone(); - let dtype = V::dtype(&data).clone(); - let len = V::len(&data); - let stats = V::stats(&data).clone(); - Ok(unsafe { Self::from_data_unchecked(vtable, dtype, len, data, stats) }) + pub fn try_new(new: ArrayParts) -> VortexResult { + new.vtable.validate(&new.data, &new.dtype, new.len)?; + Ok(unsafe { + Self::from_data_unchecked( + new.vtable, + new.dtype, + new.len, + new.data, + ArrayStats::default(), + ) + }) } /// Create without validation. @@ -140,15 +165,36 @@ pub struct Array { #[allow(clippy::same_name_method)] impl Array { - /// Create a typed array from encoding-specific data. - pub fn try_from_data(data: V::ArrayData) -> VortexResult { - let inner = ArrayRef::from_inner(Arc::new(ArrayInner::::try_from_data(data)?)); + /// Create a typed array from explicit construction parameters. + pub fn try_from_parts(new: ArrayParts) -> VortexResult { + let inner = ArrayRef::from_inner(Arc::new(ArrayInner::::try_new(new)?)); Ok(Self { inner, _phantom: PhantomData, }) } + /// Create a typed array from explicit construction parameters without validation. + /// + /// # Safety + /// Caller must ensure the provided parts are logically consistent. + #[doc(hidden)] + pub unsafe fn from_parts_unchecked(new: ArrayParts) -> Self { + let inner = ArrayRef::from_inner(Arc::new(unsafe { + ArrayInner::::from_data_unchecked( + new.vtable, + new.dtype, + new.len, + new.data, + ArrayStats::default(), + ) + })); + Self { + inner, + _phantom: PhantomData, + } + } + /// Create from an existing `ArrayRef`, trusting that it contains `ArrayInner`. /// /// # Safety @@ -203,6 +249,22 @@ impl Array { &self.downcast_inner().data } + /// Returns the full typed array construction parts. + pub fn into_parts(self) -> ArrayParts { + let inner = self.downcast_inner(); + ArrayParts { + vtable: inner.vtable.clone(), + dtype: inner.dtype.clone(), + len: inner.len, + data: inner.data.clone(), + } + } + + pub fn with_stats_set(self, stats: StatsSet) -> Self { + self.statistics().replace(stats); + self + } + /// Returns a clone of the inner encoding-specific data. pub fn into_data(self) -> V::ArrayData { self.downcast_inner().data.clone() @@ -382,3 +444,25 @@ impl From> for ArrayRef { value.inner } } + +#[cfg(test)] +mod tests { + use vortex_buffer::buffer; + + use super::Array; + use crate::arrays::Primitive; + use crate::arrays::PrimitiveArray; + use crate::assert_arrays_eq; + use crate::validity::Validity; + + #[test] + fn typed_array_into_parts_roundtrips() { + let array = PrimitiveArray::new(buffer![1i32, 2, 3], Validity::NonNullable); + let expected = array.clone(); + + let parts = array.into_parts(); + let rebuilt = Array::::try_from_parts(parts).unwrap(); + + assert_arrays_eq!(rebuilt, expected); + } +} diff --git a/vortex-array/src/array/vtable/dyn_.rs b/vortex-array/src/array/vtable/dyn_.rs index ca6cbaf0cb4..681267403e2 100644 --- a/vortex-array/src/array/vtable/dyn_.rs +++ b/vortex-array/src/array/vtable/dyn_.rs @@ -16,14 +16,12 @@ use crate::ExecutionStep; use crate::IntoArray; use crate::array::Array; use crate::array::ArrayId; -use crate::array::ArrayInner; +use crate::array::ArrayParts; use crate::array::VTable; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::executor::ExecutionCtx; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; - /// Reference-counted DynVTable pub type DynVTableRef = Arc; @@ -85,27 +83,11 @@ impl DynVTable for V { children: &dyn ArrayChildren, session: &VortexSession, ) -> VortexResult { - let metadata = V::deserialize(metadata, dtype, len, buffers, session)?; - let inner = V::build(dtype, len, &metadata, buffers, children)?; - // Validate the inner array's properties before wrapping. - assert_eq!(V::len(&inner), len, "Array length mismatch after building"); - assert_eq!( - V::dtype(&inner), - dtype, - "Array dtype mismatch after building" - ); - // Wrap in ArrayInner for safe downcasting. - // SAFETY: We just validated that V::len(&inner) == len and V::dtype(&inner) == dtype. - let array = unsafe { - ArrayInner::from_data_unchecked( - self.clone(), - dtype.clone(), - len, - inner, - ArrayStats::default(), - ) - }; - Ok(array.into_array()) + let inner = self.deserialize(dtype, len, metadata, buffers, children, session)?; + Ok( + Array::::try_from_parts(ArrayParts::new(self.clone(), dtype.clone(), len, inner))? + .into_array(), + ) } fn with_slots(&self, array: ArrayRef, slots: Vec>) -> VortexResult { @@ -114,7 +96,17 @@ impl DynVTable for V { .vortex_expect("Failed to downcast array"); let mut data = typed.data().clone(); V::with_slots(&mut data, slots)?; - Ok(Array::::try_from_data(data)?.into_array()) + let stats = array.statistics().to_owned(); + Ok(unsafe { + Array::::from_parts_unchecked(ArrayParts::new( + self.clone(), + array.dtype().clone(), + array.len(), + data, + )) + .with_stats_set(stats) + .into_array() + }) } fn reduce(&self, array: &ArrayRef) -> VortexResult> { diff --git a/vortex-array/src/array/vtable/mod.rs b/vortex-array/src/array/vtable/mod.rs index 7f233b72f5d..305cbef6ca2 100644 --- a/vortex-array/src/array/vtable/mod.rs +++ b/vortex-array/src/array/vtable/mod.rs @@ -8,6 +8,7 @@ mod operations; mod validity; use std::fmt::Debug; +use std::fmt::Formatter; use std::hash::Hasher; pub use dyn_::*; @@ -35,7 +36,6 @@ use crate::executor::ExecutionCtx; use crate::patches::Patches; use crate::scalar::ScalarValue; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; /// The array [`VTable`] encapsulates logic for an Array type within Vortex. @@ -52,28 +52,16 @@ use crate::validity::Validity; /// out of bounds). Post-conditions are validated after invocation of the vtable function and will /// panic if violated. pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { - type ArrayData: 'static + Send + Sync + Clone + Debug + IntoArray; - type Metadata: Debug; + type ArrayData: 'static + Send + Sync + Clone + Debug; type OperationsVTable: OperationsVTable; type ValidityVTable: ValidityVTable; - /// Returns the VTable from the array instance. - /// - // NOTE(ngates): this function is temporary while we migrate Arrays over to the unified vtable - fn vtable(array: &Self::ArrayData) -> &Self; - /// Returns the ID of the array. fn id(&self) -> ArrayId; - /// Returns the length of the array. - fn len(array: &Self::ArrayData) -> usize; - - /// Returns the DType of the array. - fn dtype(array: &Self::ArrayData) -> &DType; - - /// Returns the stats set for the array. - fn stats(array: &Self::ArrayData) -> &ArrayStats; + /// Validates that externally supplied logical metadata matches the array data. + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()>; /// Hashes the array contents. fn array_hash(array: &Self::ArrayData, state: &mut H, precision: Precision); @@ -130,21 +118,28 @@ pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { .vortex_expect("child_name index out of bounds") } - /// Exports metadata for an array. - fn metadata(array: ArrayView<'_, Self>) -> VortexResult; - /// Serialize metadata into a byte buffer for IPC or file storage. /// Return `None` if the array cannot be serialized. - fn serialize(metadata: Self::Metadata) -> VortexResult>>; + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>>; - /// Deserialize array metadata from a byte buffer. + /// Formats a human-readable metadata description for display tooling. + fn fmt_metadata(array: ArrayView<'_, Self>, f: &mut Formatter<'_>) -> std::fmt::Result { + match Self::serialize(array) { + Ok(Some(metadata)) if metadata.is_empty() => f.write_str("EmptyMetadata"), + _ => Debug::fmt(array.data(), f), + } + } + + /// Deserialize an array from serialized components. fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult; + &self, + dtype: &DType, + len: usize, + metadata: &[u8], + buffers: &[BufferHandle], + children: &dyn ArrayChildren, + session: &VortexSession, + ) -> VortexResult; /// Writes the array into a canonical builder. fn append_to_builder( @@ -161,15 +156,6 @@ pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { Ok(()) } - /// Build an array from components. - fn build( - dtype: &DType, - len: usize, - metadata: &Self::Metadata, - buffers: &[BufferHandle], - children: &dyn ArrayChildren, - ) -> VortexResult; - /// Returns the slots of the array as a slice. /// /// Slots provide fixed-position storage for child arrays. Each encoding defines named @@ -343,23 +329,12 @@ macro_rules! vtable { // Legacy form: FooArray is the inner struct name, no type alias generated. ($Base:ident, $VT:ident) => { $crate::aliases::paste::paste! { - impl $crate::IntoArray for [<$Base Array>] { - fn into_array(self) -> $crate::ArrayRef { - use $crate::aliases::vortex_error::VortexExpect; - $crate::ArrayRef::from($crate::Array::<$VT>::try_from_data(self).vortex_expect("data is always valid")) - } - } - - impl From<[<$Base Array>]> for $crate::ArrayRef { - fn from(value: [<$Base Array>]) -> $crate::ArrayRef { - use $crate::IntoArray; - value.into_array() - } - } - impl [<$Base Array>] { #[deprecated(note = "use `.into_array()` (owned) or `.clone().into_array()` (ref) to make clones explicit")] - pub fn to_array(&self) -> $crate::ArrayRef { + pub fn to_array(&self) -> $crate::ArrayRef + where + Self: Clone + $crate::IntoArray, + { use $crate::IntoArray; self.clone().into_array() } @@ -371,20 +346,6 @@ macro_rules! vtable { $crate::aliases::paste::paste! { /// Type alias: `FooArray = Array`. pub type [<$Base Array>] = $crate::Array<$VT>; - - impl $crate::IntoArray for $Data { - fn into_array(self) -> $crate::ArrayRef { - use $crate::aliases::vortex_error::VortexExpect; - $crate::Array::<$VT>::try_from_data(self).vortex_expect("data is always valid").into_array() - } - } - - impl From<$Data> for $crate::ArrayRef { - fn from(value: $Data) -> $crate::ArrayRef { - use $crate::IntoArray; - value.into_array() - } - } } }; } diff --git a/vortex-array/src/arrays/bool/array.rs b/vortex-array/src/arrays/bool/array.rs index 4c654a44081..688e5301c43 100644 --- a/vortex-array/src/arrays/bool/array.rs +++ b/vortex-array/src/arrays/bool/array.rs @@ -12,13 +12,14 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; use crate::array::Array; +use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Bool; use crate::arrays::BoolArray; use crate::buffer::BufferHandle; use crate::dtype::DType; -use crate::stats::ArrayStats; +use crate::dtype::Nullability; use crate::validity::Validity; /// The validity bitmap indicating which elements are non-null. @@ -62,14 +63,13 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; pub struct BoolData { /// Child arrays stored as slots. See [`VTable::slots`] for design rationale. pub(super) slots: Vec>, - pub(super) dtype: DType, + pub(super) nullability: Nullability, pub(super) bits: BufferHandle, pub(super) offset: usize, pub(super) len: usize, - pub(super) stats_set: ArrayStats, } -pub struct BoolArrayParts { +pub struct BoolDataParts { pub bits: BufferHandle, pub offset: usize, pub len: usize, @@ -84,8 +84,8 @@ impl BoolData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + DType::Bool(self.nullability) } /// Returns `true` if this array is empty. @@ -96,7 +96,7 @@ impl BoolData { /// Returns the [`Validity`] of this array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) } /// Returns the validity as a [`Mask`]. @@ -118,9 +118,9 @@ impl BoolData { /// Splits into owned parts #[inline] - pub fn into_parts(self) -> BoolArrayParts { + pub fn into_parts(self) -> BoolDataParts { let validity = self.validity(); - BoolArrayParts { + BoolDataParts { bits: self.bits, offset: self.offset, len: self.len, @@ -186,7 +186,10 @@ impl Array { /// /// Returns an error if the provided components do not satisfy the invariants. pub fn try_new(bits: BitBuffer, validity: Validity) -> VortexResult { - Array::try_from_data(BoolData::try_new(bits, validity)?) + let dtype = DType::Bool(validity.nullability()); + let len = bits.len(); + let data = BoolData::try_new(bits, validity)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Bool, dtype, len, data)) }) } /// Build a new bool array from a `BufferHandle`, returning an error if the offset is @@ -197,7 +200,9 @@ impl Array { len: usize, validity: Validity, ) -> VortexResult { - Array::try_from_data(BoolData::try_new_from_handle(bits, offset, len, validity)?) + let dtype = DType::Bool(validity.nullability()); + let data = BoolData::try_new_from_handle(bits, offset, len, validity)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Bool, dtype, len, data)) }) } /// Creates a new [`BoolArray`] without validation. @@ -206,9 +211,11 @@ impl Array { /// /// The caller must ensure that the validity length is equal to the bit buffer length. pub unsafe fn new_unchecked(bits: BitBuffer, validity: Validity) -> Self { + let dtype = DType::Bool(validity.nullability()); + let len = bits.len(); // SAFETY: caller guarantees validity length equals bit buffer length. - Array::try_from_data(unsafe { BoolData::new_unchecked(bits, validity) }) - .vortex_expect("BoolData is always valid") + let data = unsafe { BoolData::new_unchecked(bits, validity) }; + unsafe { Array::from_parts_unchecked(ArrayParts::new(Bool, dtype, len, data)) } } /// Validates the components that would be used to create a [`BoolArray`]. @@ -233,12 +240,6 @@ impl Array { pub fn into_bit_buffer(self) -> BitBuffer { self.into_data().into_bit_buffer() } - - /// Splits into owned parts, consuming self. - #[inline] - pub fn into_parts(self) -> BoolArrayParts { - self.into_data().into_parts() - } } /// Internal constructors on BoolData (used by Array constructors and VTable::build). @@ -250,15 +251,12 @@ impl BoolData { let (offset, len, buffer) = bits.into_inner(); let slots = Self::make_slots(&validity, len); - let dtype = DType::Bool(validity.nullability()); - Ok(Self { slots, - dtype, + nullability: validity.nullability(), bits: BufferHandle::new_host(buffer), offset, len, - stats_set: ArrayStats::default(), }) } @@ -284,15 +282,12 @@ impl BoolData { ); let slots = Self::make_slots(&validity, len); - let dtype = DType::Bool(validity.nullability()); - Ok(Self { slots, - dtype, + nullability: validity.nullability(), bits, offset, len, - stats_set: ArrayStats::default(), }) } @@ -302,15 +297,13 @@ impl BoolData { } else { let (offset, len, buffer) = bits.into_inner(); let slots = Self::make_slots(&validity, len); - let dtype = DType::Bool(validity.nullability()); Self { slots, - dtype, + nullability: validity.nullability(), bits: BufferHandle::new_host(buffer), offset, len, - stats_set: ArrayStats::default(), } } } diff --git a/vortex-array/src/arrays/bool/mod.rs b/vortex-array/src/arrays/bool/mod.rs index 9378983286f..b0f20794ca8 100644 --- a/vortex-array/src/arrays/bool/mod.rs +++ b/vortex-array/src/arrays/bool/mod.rs @@ -4,8 +4,8 @@ mod array; mod patch; -pub use array::BoolArrayParts; pub use array::BoolData; +pub use array::BoolDataParts; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/bool/vtable/mod.rs b/vortex-array/src/arrays/bool/vtable/mod.rs index 17a87a92cbe..c2a5c25ca8c 100644 --- a/vortex-array/src/arrays/bool/vtable/mod.rs +++ b/vortex-array/src/arrays/bool/vtable/mod.rs @@ -1,7 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::fmt::Formatter; + use kernel::PARENT_KERNELS; +use prost::Message; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -10,11 +13,8 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; -use crate::DeserializeMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; -use crate::ProstMetadata; -use crate::SerializeMetadata; use crate::array::Array; use crate::array::ArrayView; use crate::array::VTable; @@ -36,7 +36,6 @@ use crate::array::ArrayId; use crate::arrays::bool::compute::rules::RULES; use crate::hash::ArrayEq; use crate::hash::ArrayHash; -use crate::stats::ArrayStats; vtable!(Bool, Bool, BoolData); @@ -50,30 +49,13 @@ pub struct BoolMetadata { impl VTable for Bool { type ArrayData = BoolData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Bool - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &BoolData) -> usize { - array.len - } - - fn dtype(array: &BoolData) -> &DType { - &array.dtype - } - - fn stats(array: &BoolData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &BoolData, state: &mut H, precision: Precision) { array.to_bit_buffer().array_hash(state, precision); array.validity().array_hash(state, precision); @@ -104,35 +86,50 @@ impl VTable for Bool { } } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { assert!(array.offset < 8, "Offset must be <8, got {}", array.offset); - Ok(ProstMetadata(BoolMetadata { - offset: u32::try_from(array.offset).vortex_expect("checked"), - })) + Ok(Some( + BoolMetadata { + offset: u32::try_from(array.offset).vortex_expect("checked"), + } + .encode_to_vec(), + )) } - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + fn fmt_metadata(array: ArrayView<'_, Self>, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "BoolMetadata {{ offset: {} }}", array.offset) } - fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - let metadata = ::deserialize(bytes)?; - Ok(ProstMetadata(metadata)) + fn validate(&self, data: &BoolData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.len() == len, + "BoolArray length {} does not match outer length {}", + data.len(), + len + ); + + let actual_dtype = data.dtype(); + vortex_ensure!( + &actual_dtype == dtype, + "BoolArray dtype {} does not match outer dtype {}", + actual_dtype, + dtype + ); + + Ok(()) } - fn build( + fn deserialize( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = BoolMetadata::decode(metadata)?; if buffers.len() != 1 { vortex_bail!("Expected 1 buffer, got {}", buffers.len()); } @@ -209,8 +206,8 @@ mod tests { use crate::LEGACY_SESSION; use crate::arrays::BoolArray; use crate::assert_arrays_eq; - use crate::serde::ArrayParts; use crate::serde::SerializeOptions; + use crate::serde::SerializedArray; #[test] fn test_nullable_bool_serde_roundtrip() { @@ -229,7 +226,7 @@ mod tests { for buf in serialized { concat.extend_from_slice(buf.as_ref()); } - let parts = ArrayParts::try_from(concat.freeze()).unwrap(); + let parts = SerializedArray::try_from(concat.freeze()).unwrap(); let decoded = parts .decode( &dtype, diff --git a/vortex-array/src/arrays/chunked/array.rs b/vortex-array/src/arrays/chunked/array.rs index 5aa15ce6d57..33b471e1852 100644 --- a/vortex-array/src/arrays/chunked/array.rs +++ b/vortex-array/src/arrays/chunked/array.rs @@ -17,6 +17,7 @@ use vortex_error::vortex_bail; use crate::ArrayRef; use crate::IntoArray; use crate::array::Array; +use crate::array::ArrayParts; use crate::arrays::Chunked; use crate::arrays::primitive::PrimitiveData; use crate::dtype::DType; @@ -24,7 +25,6 @@ use crate::iter::ArrayIterator; use crate::iter::ArrayIteratorAdapter; use crate::search_sorted::SearchSorted; use crate::search_sorted::SearchSortedSide; -use crate::stats::ArrayStats; use crate::stream::ArrayStream; use crate::stream::ArrayStreamAdapter; use crate::validity::Validity; @@ -34,12 +34,10 @@ pub(super) const CHUNKS_OFFSET: usize = 1; #[derive(Clone, Debug)] pub struct ChunkedData { - pub(super) dtype: DType, - pub(super) len: usize, + pub(super) empty_dtype: Option, pub(super) chunk_offsets: PrimitiveData, pub(super) chunks: Vec, pub(super) slots: Vec>, - pub(super) stats_set: ArrayStats, } impl ChunkedData { @@ -49,7 +47,17 @@ impl ChunkedData { chunks: &[ArrayRef], ) -> Vec> { let mut slots = Vec::with_capacity(1 + chunks.len()); - slots.push(Some(chunk_offsets.clone().into_array())); + slots.push(Some( + unsafe { + Array::from_parts_unchecked(ArrayParts::new( + crate::arrays::Primitive, + chunk_offsets.dtype(), + chunk_offsets.len(), + chunk_offsets.clone(), + )) + } + .into_array(), + )); slots.extend(chunks.iter().map(|c| Some(c.clone()))); slots } @@ -98,14 +106,10 @@ impl ChunkedData { let slots = Self::make_slots(&chunk_offsets, &chunks); Self { - dtype, - len: curr_offset - .try_into() - .vortex_expect("chunk offset must fit in usize"), + empty_dtype: chunks.is_empty().then_some(dtype), chunk_offsets, chunks, slots, - stats_set: Default::default(), } } @@ -124,12 +128,20 @@ impl ChunkedData { /// Returns the length of this array. pub fn len(&self) -> usize { - self.len + self.chunk_offsets() + .last() + .copied() + .and_then(|len| usize::try_from(len).ok()) + .vortex_expect("chunk offset must fit in usize") } /// Returns the [`DType`] of this array. pub fn dtype(&self) -> &DType { - &self.dtype + self.chunks + .first() + .map(|chunk| chunk.dtype()) + .or(self.empty_dtype.as_ref()) + .vortex_expect("ChunkedArray dtype must come from chunks or the empty dtype") } /// Returns `true` if this array is empty. @@ -219,10 +231,12 @@ impl ChunkedData { new_chunks.push( // SAFETY: chunks_to_combine contains valid chunks of the same dtype as self. // All chunks are guaranteed to be valid arrays matching self.dtype(). - unsafe { ChunkedData::new_unchecked(chunks_to_combine, self.dtype().clone()) } - .into_array() - .to_canonical()? - .into_array(), + unsafe { + Array::::new_unchecked(chunks_to_combine, self.dtype().clone()) + } + .into_array() + .to_canonical()? + .into_array(), ); new_chunk_n_bytes = 0; @@ -243,7 +257,7 @@ impl ChunkedData { new_chunks.push( // SAFETY: chunks_to_combine contains valid chunks of the same dtype as self. // All chunks are guaranteed to be valid arrays matching self.dtype(). - unsafe { ChunkedData::new_unchecked(chunks_to_combine, self.dtype().clone()) } + unsafe { Array::::new_unchecked(chunks_to_combine, self.dtype().clone()) } .into_array() .to_canonical()? .into_array(), @@ -259,7 +273,22 @@ impl ChunkedData { impl Array { /// Constructs a new `ChunkedArray`. pub fn try_new(chunks: Vec, dtype: DType) -> VortexResult { - Array::try_from_data(ChunkedData::try_new(chunks, dtype)?) + let data = ChunkedData::try_new(chunks, dtype)?; + let dtype = data.dtype().clone(); + let len = data.len(); + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Chunked, dtype, len, data)) }) + } + + pub fn rechunk(&self, target_bytesize: u64, target_rowsize: usize) -> VortexResult { + let data = self.data().rechunk(target_bytesize, target_rowsize)?; + Ok(unsafe { + Array::from_parts_unchecked(ArrayParts::new( + Chunked, + self.dtype().clone(), + data.len(), + data, + )) + }) } /// Creates a new `ChunkedArray` without validation. @@ -268,8 +297,10 @@ impl Array { /// /// See [`ChunkedData::new_unchecked`]. pub unsafe fn new_unchecked(chunks: Vec, dtype: DType) -> Self { - Array::try_from_data(unsafe { ChunkedData::new_unchecked(chunks, dtype) }) - .vortex_expect("ChunkedData is always valid") + let data = unsafe { ChunkedData::new_unchecked(chunks, dtype) }; + let dtype = data.dtype().clone(); + let len = data.len(); + unsafe { Array::from_parts_unchecked(ArrayParts::new(Chunked, dtype, len, data)) } } } diff --git a/vortex-array/src/arrays/chunked/tests.rs b/vortex-array/src/arrays/chunked/tests.rs index 8eefe189a29..f1266a5373d 100644 --- a/vortex-array/src/arrays/chunked/tests.rs +++ b/vortex-array/src/arrays/chunked/tests.rs @@ -8,6 +8,7 @@ use vortex_buffer::buffer; use crate::IntoArray; use crate::accessor::ArrayAccessor; +use crate::array::ArrayParts; use crate::array::VTable; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; @@ -207,10 +208,12 @@ fn with_slots_updates_nchunks_len_and_offsets() { ]; let expected_nchunks = slots.len() - 1; let expected_len = orig.len(); + let dtype = orig.dtype().clone(); let mut data = orig.into_data(); ::with_slots(&mut data, slots).unwrap(); - let array = ChunkedArray::try_from_data(data).unwrap(); + let array = + ChunkedArray::try_from_parts(ArrayParts::new(Chunked, dtype, expected_len, data)).unwrap(); assert_eq!(array.nchunks(), expected_nchunks); assert_eq!(array.len(), expected_len); diff --git a/vortex-array/src/arrays/chunked/vtable/mod.rs b/vortex-array/src/arrays/chunked/vtable/mod.rs index 3f20f9d56ea..6349bfa97ec 100644 --- a/vortex-array/src/arrays/chunked/vtable/mod.rs +++ b/vortex-array/src/arrays/chunked/vtable/mod.rs @@ -11,7 +11,6 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::Canonical; -use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::IntoArray; @@ -21,6 +20,7 @@ use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; use crate::array::VTable; +use crate::arrays::PrimitiveArray; use crate::arrays::chunked::ChunkedData; use crate::arrays::chunked::array::CHUNK_OFFSETS_SLOT; use crate::arrays::chunked::array::CHUNKS_OFFSET; @@ -36,7 +36,6 @@ use crate::dtype::PType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; mod canonical; @@ -54,47 +53,39 @@ impl Chunked { impl VTable for Chunked { type ArrayData = ChunkedData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Chunked - } fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ChunkedData) -> usize { - array.len - } - - fn dtype(array: &ChunkedData) -> &DType { - &array.dtype - } - - fn stats(array: &ChunkedData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &ChunkedData, state: &mut H, precision: Precision) { - array - .chunk_offsets - .clone() - .into_array() - .array_hash(state, precision); + PrimitiveArray::new( + array.chunk_offsets.to_buffer::(), + Validity::NonNullable, + ) + .into_array() + .array_hash(state, precision); for chunk in &array.chunks { chunk.array_hash(state, precision); } } fn array_eq(array: &ChunkedData, other: &ChunkedData, precision: Precision) -> bool { - array - .chunk_offsets - .clone() - .into_array() - .array_eq(&other.chunk_offsets.clone().into_array(), precision) - && array.chunks.len() == other.chunks.len() + PrimitiveArray::new( + array.chunk_offsets.to_buffer::(), + Validity::NonNullable, + ) + .into_array() + .array_eq( + &PrimitiveArray::new( + other.chunk_offsets.to_buffer::(), + Validity::NonNullable, + ) + .into_array(), + precision, + ) && array.chunks.len() == other.chunks.len() && array .iter_chunks() .zip(other.iter_chunks()) @@ -113,31 +104,42 @@ impl VTable for Chunked { vortex_panic!("ChunkedArray buffer_name index {idx} out of bounds") } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } - fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) + fn validate(&self, data: &ChunkedData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.len() == len, + "ChunkedArray length {} does not match outer length {}", + data.len(), + len + ); + vortex_ensure!( + data.dtype() == dtype, + "ChunkedArray dtype {} does not match outer dtype {}", + data.dtype(), + dtype + ); + Ok(()) } - fn build( + fn deserialize( + &self, dtype: &DType, _len: usize, - _metadata: &Self::Metadata, + metadata: &[u8], + _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + if !metadata.is_empty() { + vortex_bail!( + "ChunkedArray expects empty metadata, got {} bytes", + metadata.len() + ); + } if children.is_empty() { vortex_bail!("Chunked array needs at least one child"); } @@ -168,23 +170,15 @@ impl VTable for Chunked { }) .try_collect()?; - let chunk_offsets = PrimitiveData::new(chunk_offsets_buf.clone(), Validity::NonNullable); - - let total_len = chunk_offsets_buf - .last() - .ok_or_else(|| vortex_err!("chunk_offsets must not be empty"))?; - let len = usize::try_from(*total_len) - .map_err(|_| vortex_err!("total length {} exceeds usize range", total_len))?; + let chunk_offsets = PrimitiveData::new(chunk_offsets_buf, Validity::NonNullable); let slots = ChunkedData::make_slots(&chunk_offsets, &chunks); // Construct directly using the struct fields to avoid recomputing chunk_offsets Ok(ChunkedData { - dtype: dtype.clone(), - len, + empty_dtype: chunks.is_empty().then_some(dtype.clone()), chunk_offsets, chunks, slots, - stats_set: Default::default(), }) } @@ -223,16 +217,11 @@ impl VTable for Chunked { .ok_or_else(|| vortex_err!("chunk slot must not be None")) }) .try_collect()?; - array.chunk_offsets = PrimitiveData::new(chunk_offsets_buf.clone(), Validity::NonNullable); + array.chunk_offsets = PrimitiveData::new(chunk_offsets_buf, Validity::NonNullable); + array.empty_dtype = chunks.is_empty().then_some(array.dtype().clone()); array.chunks = chunks; array.slots = slots; - let total_len = chunk_offsets_buf - .last() - .ok_or_else(|| vortex_err!("chunk_offsets must not be empty"))?; - array.len = usize::try_from(*total_len) - .map_err(|_| vortex_err!("total length {} exceeds usize range", total_len))?; - Ok(()) } diff --git a/vortex-array/src/arrays/chunked/vtable/validity.rs b/vortex-array/src/arrays/chunked/vtable/validity.rs index 18e429b46b0..a250c98be89 100644 --- a/vortex-array/src/arrays/chunked/vtable/validity.rs +++ b/vortex-array/src/arrays/chunked/vtable/validity.rs @@ -8,7 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::array::ValidityVTable; use crate::arrays::Chunked; -use crate::arrays::chunked::ChunkedData; +use crate::arrays::ChunkedArray; use crate::dtype::DType; use crate::dtype::Nullability; use crate::validity::Validity; @@ -39,7 +39,7 @@ impl ValidityVTable for Chunked { Ok(Validity::Array( unsafe { - ChunkedData::new_unchecked( + ChunkedArray::new_unchecked( validities .into_iter() .zip(array.iter_chunks()) diff --git a/vortex-array/src/arrays/constant/array.rs b/vortex-array/src/arrays/constant/array.rs index ff32124210a..3841a9d2b02 100644 --- a/vortex-array/src/arrays/constant/array.rs +++ b/vortex-array/src/arrays/constant/array.rs @@ -1,54 +1,32 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use vortex_error::VortexExpect; - use crate::ArrayRef; use crate::array::Array; +use crate::array::ArrayParts; use crate::arrays::Constant; -use crate::dtype::DType; use crate::scalar::Scalar; -use crate::stats::ArrayStats; pub(super) const NUM_SLOTS: usize = 0; #[derive(Clone, Debug)] pub struct ConstantData { pub(super) scalar: Scalar, - pub(super) len: usize, pub(super) slots: Vec>, - pub(super) stats_set: ArrayStats, } impl ConstantData { - pub fn new(scalar: S, len: usize) -> Self + pub fn new(scalar: S) -> Self where S: Into, { let scalar = scalar.into(); Self { scalar, - len, slots: vec![], - stats_set: Default::default(), } } - /// Returns the length of this array. - pub fn len(&self) -> usize { - self.len - } - - /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - self.scalar.dtype() - } - - /// Returns `true` if this array is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - /// Returns the [`Scalar`] value of this constant array. pub fn scalar(&self) -> &Scalar { &self.scalar @@ -64,7 +42,9 @@ impl Array { where S: Into, { - Array::try_from_data(ConstantData::new(scalar, len)) - .vortex_expect("ConstantData is always valid") + let scalar = scalar.into(); + let dtype = scalar.dtype().clone(); + let data = ConstantData::new(scalar); + unsafe { Array::from_parts_unchecked(ArrayParts::new(Constant, dtype, len, data)) } } } diff --git a/vortex-array/src/arrays/constant/compute/take.rs b/vortex-array/src/arrays/constant/compute/take.rs index 4bbd44132e1..96aa71fd9ac 100644 --- a/vortex-array/src/arrays/constant/compute/take.rs +++ b/vortex-array/src/arrays/constant/compute/take.rs @@ -9,9 +9,9 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Constant; use crate::arrays::ConstantArray; +use crate::arrays::MaskedArray; use crate::arrays::dict::TakeReduce; use crate::arrays::dict::TakeReduceAdaptor; -use crate::arrays::masked::MaskedData; use crate::optimizer::rules::ParentRuleSet; use crate::scalar::Scalar; use crate::validity::Validity; @@ -45,7 +45,7 @@ impl TakeReduce for Constant { return Ok(Some(arr)); } - MaskedData::try_new(arr, Validity::from(v.clone()))?.into_array() + MaskedArray::try_new(arr, Validity::from(v.clone()))?.into_array() } }; Ok(Some(result)) diff --git a/vortex-array/src/arrays/constant/vtable/mod.rs b/vortex-array/src/arrays/constant/vtable/mod.rs index d6fd4407635..55869cbd220 100644 --- a/vortex-array/src/arrays/constant/vtable/mod.rs +++ b/vortex-array/src/arrays/constant/vtable/mod.rs @@ -39,7 +39,6 @@ use crate::scalar::DecimalValue; use crate::scalar::Scalar; use crate::scalar::ScalarValue; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::vtable; pub(crate) mod canonical; mod operations; @@ -57,28 +56,19 @@ impl Constant { impl VTable for Constant { type ArrayData = ConstantData; - type Metadata = Scalar; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Constant - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ConstantData) -> usize { - array.len - } - - fn dtype(array: &ConstantData) -> &DType { - array.scalar.dtype() - } - - fn stats(array: &ConstantData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &ConstantData, dtype: &DType, _len: usize) -> VortexResult<()> { + vortex_ensure!( + data.scalar.dtype() == dtype, + "ConstantArray scalar dtype does not match outer dtype" + ); + Ok(()) } fn array_hash( @@ -131,23 +121,22 @@ impl VTable for Constant { Ok(()) } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(array.scalar().clone()) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { // HACK: Because the scalar is stored in the buffers, we do not need to serialize the // metadata at all. Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], + &self, dtype: &DType, _len: usize, + _metadata: &[u8], + buffers: &[BufferHandle], + _children: &dyn ArrayChildren, session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult { vortex_ensure!( buffers.len() == 1, "Expected 1 buffer, got {}", @@ -160,17 +149,7 @@ impl VTable for Constant { let scalar_value = ScalarValue::from_proto_bytes(bytes, dtype, session)?; let scalar = Scalar::try_new(dtype.clone(), scalar_value)?; - Ok(scalar) - } - - fn build( - _dtype: &DType, - len: usize, - metadata: &Self::Metadata, - _buffers: &[BufferHandle], - _children: &dyn ArrayChildren, - ) -> VortexResult { - Ok(ConstantData::new(metadata.clone(), len)) + Ok(ConstantData::new(scalar)) } fn reduce_parent( diff --git a/vortex-array/src/arrays/decimal/array.rs b/vortex-array/src/arrays/decimal/array.rs index f8e3ce0c976..a51657fd958 100644 --- a/vortex-array/src/arrays/decimal/array.rs +++ b/vortex-array/src/arrays/decimal/array.rs @@ -15,6 +15,7 @@ use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; use crate::array::Array; +use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Decimal; @@ -27,10 +28,10 @@ use crate::dtype::DecimalDType; use crate::dtype::DecimalType; use crate::dtype::IntegerPType; use crate::dtype::NativeDecimalType; +use crate::dtype::Nullability; use crate::match_each_decimal_value_type; use crate::match_each_integer_ptype; use crate::patches::Patches; -use crate::stats::ArrayStats; use crate::validity::Validity; /// The validity bitmap indicating which elements are non-null. @@ -98,13 +99,13 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; #[derive(Clone, Debug)] pub struct DecimalData { pub(super) slots: Vec>, - pub(super) dtype: DType, + pub(super) decimal_dtype: DecimalDType, + pub(super) nullability: Nullability, pub(super) values: BufferHandle, pub(super) values_type: DecimalType, - pub(super) stats_set: ArrayStats, } -pub struct DecimalArrayParts { +pub struct DecimalDataParts { pub decimal_dtype: DecimalDType, pub values: BufferHandle, pub values_type: DecimalType, @@ -242,8 +243,8 @@ impl DecimalData { slots: Self::make_slots(&validity, len), values, values_type, - dtype: DType::Decimal(decimal_dtype, validity.nullability()), - stats_set: Default::default(), + decimal_dtype, + nullability: validity.nullability(), } } @@ -300,8 +301,8 @@ impl DecimalData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + DType::Decimal(self.decimal_dtype, self.nullability) } /// Returns `true` if this array is empty. @@ -311,15 +312,13 @@ impl DecimalData { /// Reconstructs the validity from the slot state. pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) } - pub fn into_parts(self) -> DecimalArrayParts { + pub fn into_parts(self) -> DecimalDataParts { let validity = self.validity(); - let decimal_dtype = self.dtype.into_decimal_opt().vortex_expect("cannot fail"); - - DecimalArrayParts { - decimal_dtype, + DecimalDataParts { + decimal_dtype: self.decimal_dtype, values: self.values, values_type: self.values_type, validity, @@ -344,11 +343,7 @@ impl DecimalData { /// Returns the decimal type information pub fn decimal_dtype(&self) -> DecimalDType { - if let DType::Decimal(decimal_dtype, _) = self.dtype { - decimal_dtype - } else { - vortex_panic!("Expected Decimal dtype, got {:?}", self.dtype) - } + self.decimal_dtype } /// Return the `DecimalType` used to represent the values in the array. @@ -450,8 +445,10 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> Self { - Array::try_from_data(DecimalData::new(buffer, decimal_dtype, validity)) - .vortex_expect("DecimalData is always valid") + let dtype = DType::Decimal(decimal_dtype, validity.nullability()); + let len = buffer.len(); + let data = DecimalData::new(buffer, decimal_dtype, validity); + unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } } /// Creates a new [`DecimalArray`] without validation. @@ -464,8 +461,10 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> Self { - Array::try_from_data(unsafe { DecimalData::new_unchecked(buffer, decimal_dtype, validity) }) - .vortex_expect("DecimalData is always valid") + let dtype = DType::Decimal(decimal_dtype, validity.nullability()); + let len = buffer.len(); + let data = unsafe { DecimalData::new_unchecked(buffer, decimal_dtype, validity) }; + unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } } /// Creates a new [`DecimalArray`] from a host-native buffer with validation. @@ -474,7 +473,10 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> VortexResult { - Array::try_from_data(DecimalData::try_new(buffer, decimal_dtype, validity)?) + let dtype = DType::Decimal(decimal_dtype, validity.nullability()); + let len = buffer.len(); + let data = DecimalData::try_new(buffer, decimal_dtype, validity)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) }) } /// Creates a new [`DecimalArray`] from an iterator of values. @@ -486,8 +488,10 @@ impl Array { iter: I, decimal_dtype: DecimalDType, ) -> Self { - Array::try_from_data(DecimalData::from_iter(iter, decimal_dtype)) - .vortex_expect("DecimalData is always valid") + let data = DecimalData::from_iter(iter, decimal_dtype); + let dtype = data.dtype(); + let len = data.len(); + unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } } /// Creates a new [`DecimalArray`] from an iterator of optional values. @@ -495,8 +499,10 @@ impl Array { iter: I, decimal_dtype: DecimalDType, ) -> Self { - Array::try_from_data(DecimalData::from_option_iter(iter, decimal_dtype)) - .vortex_expect("DecimalData is always valid") + let data = DecimalData::from_option_iter(iter, decimal_dtype); + let dtype = data.dtype(); + let len = data.len(); + unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } } /// Creates a new [`DecimalArray`] from a [`BufferHandle`]. @@ -506,13 +512,10 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> Self { - Array::try_from_data(DecimalData::new_handle( - values, - values_type, - decimal_dtype, - validity, - )) - .vortex_expect("DecimalData is always valid") + let dtype = DType::Decimal(decimal_dtype, validity.nullability()); + let len = values.len() / values_type.byte_width(); + let data = DecimalData::new_handle(values, values_type, decimal_dtype, validity); + unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } } /// Creates a new [`DecimalArray`] without validation from a [`BufferHandle`]. @@ -526,10 +529,12 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> Self { - Array::try_from_data(unsafe { + let dtype = DType::Decimal(decimal_dtype, validity.nullability()); + let len = values.len() / values_type.byte_width(); + let data = unsafe { DecimalData::new_unchecked_handle(values, values_type, decimal_dtype, validity) - }) - .vortex_expect("DecimalData is always valid") + }; + unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } } } diff --git a/vortex-array/src/arrays/decimal/compute/between.rs b/vortex-array/src/arrays/decimal/compute/between.rs index fd8627c898c..b521d2148e0 100644 --- a/vortex-array/src/arrays/decimal/compute/between.rs +++ b/vortex-array/src/arrays/decimal/compute/between.rs @@ -36,7 +36,7 @@ impl BetweenKernel for Decimal { // NOTE: we know that have checked before that the lower and upper bounds are not all null. let nullability = - arr.dtype.nullability() | lower.dtype().nullability() | upper.dtype().nullability(); + arr.dtype().nullability() | lower.dtype().nullability() | upper.dtype().nullability(); match_each_decimal_value_type!(arr.values_type(), |D| { between_unpack::(arr, lower, upper, nullability, options) diff --git a/vortex-array/src/arrays/decimal/mod.rs b/vortex-array/src/arrays/decimal/mod.rs index 137331a492f..a0b90613f52 100644 --- a/vortex-array/src/arrays/decimal/mod.rs +++ b/vortex-array/src/arrays/decimal/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::DecimalArrayParts; pub use array::DecimalData; +pub use array::DecimalDataParts; pub use vtable::DecimalArray; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/decimal/vtable/mod.rs b/vortex-array/src/arrays/decimal/vtable/mod.rs index 922b1b80fe2..dc7a46c779d 100644 --- a/vortex-array/src/arrays/decimal/vtable/mod.rs +++ b/vortex-array/src/arrays/decimal/vtable/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use kernel::PARENT_KERNELS; +use prost::Message; use vortex_buffer::Alignment; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -10,11 +11,8 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; -use crate::DeserializeMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; -use crate::ProstMetadata; -use crate::SerializeMetadata; use crate::array::Array; use crate::array::ArrayView; use crate::array::VTable; @@ -40,7 +38,6 @@ use crate::arrays::decimal::array::SLOT_NAMES; use crate::arrays::decimal::compute::rules::RULES; use crate::hash::ArrayEq; use crate::hash::ArrayHash; -use crate::stats::ArrayStats; vtable!(Decimal, Decimal, DecimalData); // The type of the values can be determined by looking at the type info...right? @@ -53,38 +50,13 @@ pub struct DecimalMetadata { impl VTable for Decimal { type ArrayData = DecimalData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Decimal - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &DecimalData) -> usize { - let divisor = match array.values_type { - DecimalType::I8 => 1, - DecimalType::I16 => 2, - DecimalType::I32 => 4, - DecimalType::I64 => 8, - DecimalType::I128 => 16, - DecimalType::I256 => 32, - }; - array.values.len() / divisor - } - - fn dtype(array: &DecimalData) -> &DType { - &array.dtype - } - - fn stats(array: &DecimalData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &DecimalData, state: &mut H, precision: Precision) { array.values.array_hash(state, precision); std::mem::discriminant(&array.values_type).hash(state); @@ -115,34 +87,45 @@ impl VTable for Decimal { } } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(DecimalMetadata { - values_type: array.values_type() as i32, - })) + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some( + DecimalMetadata { + values_type: array.values_type() as i32, + } + .encode_to_vec(), + )) } - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) - } + fn validate(&self, data: &DecimalData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.len() == len, + "DecimalArray length {} does not match outer length {}", + data.len(), + len + ); - fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - let metadata = ProstMetadata::::deserialize(bytes)?; - Ok(ProstMetadata(metadata)) + let actual_dtype = data.dtype(); + vortex_ensure!( + &actual_dtype == dtype, + "DecimalArray dtype {} does not match outer dtype {}", + actual_dtype, + dtype + ); + + Ok(()) } - fn build( + fn deserialize( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = DecimalMetadata::decode(metadata)?; if buffers.len() != 1 { vortex_bail!("Expected 1 buffer, got {}", buffers.len()); } @@ -233,8 +216,8 @@ mod tests { use crate::arrays::DecimalArray; use crate::assert_arrays_eq; use crate::dtype::DecimalDType; - use crate::serde::ArrayParts; use crate::serde::SerializeOptions; + use crate::serde::SerializedArray; use crate::validity::Validity; #[test] @@ -259,7 +242,7 @@ mod tests { let concat = concat.freeze(); - let parts = ArrayParts::try_from(concat).unwrap(); + let parts = SerializedArray::try_from(concat).unwrap(); let decoded = parts .decode(&dtype, 5, &ReadContext::new(ctx.to_ids()), &LEGACY_SESSION) .unwrap(); @@ -287,7 +270,7 @@ mod tests { concat.extend_from_slice(buf.as_ref()); } - let parts = ArrayParts::try_from(concat.freeze()).unwrap(); + let parts = SerializedArray::try_from(concat.freeze()).unwrap(); let decoded = parts .decode( &dtype, diff --git a/vortex-array/src/arrays/dict/array.rs b/vortex-array/src/arrays/dict/array.rs index ceff69fa242..a791d79b798 100644 --- a/vortex-array/src/arrays/dict/array.rs +++ b/vortex-array/src/arrays/dict/array.rs @@ -11,11 +11,11 @@ use vortex_mask::AllOr; use crate::ArrayRef; use crate::ToCanonical; use crate::array::Array; +use crate::array::ArrayParts; use crate::arrays::Dict; use crate::dtype::DType; use crate::dtype::PType; use crate::match_each_integer_ptype; -use crate::stats::ArrayStats; #[derive(Clone, prost::Message)] pub struct DictMetadata { @@ -43,8 +43,6 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["codes", "values"]; #[derive(Debug, Clone)] pub struct DictData { pub(super) slots: Vec>, - pub(super) stats_set: ArrayStats, - pub(super) dtype: DType, /// Indicates whether all dictionary values are definitely referenced by at least one code. /// `true` = all values are referenced (computed during encoding). /// `false` = unknown/might have unreferenced values. @@ -53,10 +51,9 @@ pub struct DictData { pub(super) all_values_referenced: bool, } -pub struct DictArrayParts { +pub struct DictDataParts { pub codes: ArrayRef, pub values: ArrayRef, - pub dtype: DType, } impl DictData { @@ -67,13 +64,8 @@ impl DictData { /// by the safe `DictArray::try_new` constructor are valid, for example when /// you are filtering or slicing an existing valid `DictArray`. pub unsafe fn new_unchecked(codes: ArrayRef, values: ArrayRef) -> Self { - let dtype = values - .dtype() - .union_nullability(codes.dtype().nullability()); Self { slots: vec![Some(codes), Some(values)], - stats_set: Default::default(), - dtype, all_values_referenced: false, } } @@ -119,7 +111,7 @@ impl DictData { /// of the `values` array. Otherwise, this constructor returns an error. /// /// It is an error to provide a nullable `codes` with non-nullable `values`. - pub fn try_new(codes: ArrayRef, values: ArrayRef) -> VortexResult { + pub(crate) fn try_new(codes: ArrayRef, values: ArrayRef) -> VortexResult { if !codes.dtype().is_int() { vortex_bail!(MismatchedTypes: "int", codes.dtype()); } @@ -133,8 +125,10 @@ impl DictData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + self.values() + .dtype() + .union_nullability(self.codes().dtype().nullability()) } /// Returns `true` if this array is empty. @@ -142,15 +136,14 @@ impl DictData { self.len() == 0 } - pub fn into_parts(mut self) -> DictArrayParts { - DictArrayParts { + pub fn into_parts(mut self) -> DictDataParts { + DictDataParts { codes: self.slots[CODES_SLOT] .take() .vortex_expect("DictArray codes slot"), values: self.slots[VALUES_SLOT] .take() .vortex_expect("DictArray values slot"), - dtype: self.dtype, } } @@ -204,12 +197,18 @@ impl DictData { impl Array { /// Build a new `DictArray` from its components, `codes` and `values`. pub fn new(codes: ArrayRef, values: ArrayRef) -> Self { - Array::try_from_data(DictData::new(codes, values)).vortex_expect("DictData is always valid") + let data = DictData::new(codes, values); + let dtype = data.dtype(); + let len = data.len(); + unsafe { Array::from_parts_unchecked(ArrayParts::new(Dict, dtype, len, data)) } } /// Build a new `DictArray` from its components, `codes` and `values`. pub fn try_new(codes: ArrayRef, values: ArrayRef) -> VortexResult { - Array::try_from_data(DictData::try_new(codes, values)?) + let data = DictData::try_new(codes, values)?; + let dtype = data.dtype(); + let len = data.len(); + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Dict, dtype, len, data)) }) } /// Build a new `DictArray` without validating the codes or values. @@ -218,8 +217,10 @@ impl Array { /// /// See [`DictData::new_unchecked`]. pub unsafe fn new_unchecked(codes: ArrayRef, values: ArrayRef) -> Self { - Array::try_from_data(unsafe { DictData::new_unchecked(codes, values) }) - .vortex_expect("DictData is always valid") + let data = unsafe { DictData::new_unchecked(codes, values) }; + let dtype = data.dtype(); + let len = data.len(); + unsafe { Array::from_parts_unchecked(ArrayParts::new(Dict, dtype, len, data)) } } /// Set whether all values in the dictionary are referenced by at least one code. @@ -228,11 +229,13 @@ impl Array { /// /// See [`DictData::set_all_values_referenced`]. pub unsafe fn set_all_values_referenced(self, all_values_referenced: bool) -> Self { - Array::try_from_data(unsafe { + let dtype = self.dtype().clone(); + let len = self.len(); + let data = unsafe { self.into_data() .set_all_values_referenced(all_values_referenced) - }) - .vortex_expect("data is always valid") + }; + unsafe { Array::from_parts_unchecked(ArrayParts::new(Dict, dtype, len, data)) } } } @@ -442,18 +445,20 @@ mod test { #[cfg_attr(miri, ignore)] #[test] fn test_dict_metadata() { + use prost::Message; + use super::DictMetadata; - use crate::ProstMetadata; use crate::test_harness::check_metadata; check_metadata( "dict.metadata", - ProstMetadata(DictMetadata { + &DictMetadata { codes_ptype: PType::U64 as i32, values_len: u32::MAX, is_nullable_codes: None, all_values_referenced: None, - }), + } + .encode_to_vec(), ); } } diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index f65e2d43412..25be2053332 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use kernel::PARENT_KERNELS; +use prost::Message; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -10,8 +11,8 @@ use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; -use super::DictArrayParts; use super::DictData; +use super::DictDataParts; use super::DictMetadata; use super::array::NUM_SLOTS; use super::array::SLOT_NAMES; @@ -19,16 +20,13 @@ use super::take_canonical; use crate::AnyCanonical; use crate::ArrayRef; use crate::Canonical; -use crate::DeserializeMetadata; use crate::Precision; -use crate::ProstMetadata; -use crate::SerializeMetadata; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; use crate::array::VTable; +use crate::arrays::ConstantArray; use crate::arrays::Primitive; -use crate::arrays::constant::ConstantData; use crate::arrays::dict::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -41,7 +39,6 @@ use crate::hash::ArrayHash; use crate::require_child; use crate::scalar::Scalar; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::vtable; mod kernel; mod operations; @@ -59,28 +56,20 @@ impl Dict { impl VTable for Dict { type ArrayData = DictData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Dict - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &DictData) -> usize { - array.codes().len() - } - - fn dtype(array: &DictData) -> &DType { - &array.dtype - } - - fn stats(array: &DictData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &DictData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!(data.codes().len() == len, "DictArray codes length mismatch"); + vortex_ensure!( + data.dtype() == *dtype, + "DictArray dtype does not match codes/values dtype" + ); + Ok(()) } fn array_hash(array: &DictData, state: &mut H, precision: Precision) { @@ -105,42 +94,34 @@ impl VTable for Dict { None } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(DictMetadata { - codes_ptype: PType::try_from(array.codes().dtype())? as i32, - values_len: u32::try_from(array.values().len()).map_err(|_| { - vortex_err!( - "Dictionary values size {} overflowed u32", - array.values().len() - ) - })?, - is_nullable_codes: Some(array.codes().dtype().is_nullable()), - all_values_referenced: Some(array.all_values_referenced), - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some( + DictMetadata { + codes_ptype: PType::try_from(array.codes().dtype())? as i32, + values_len: u32::try_from(array.values().len()).map_err(|_| { + vortex_err!( + "Dictionary values size {} overflowed u32", + array.values().len() + ) + })?, + is_nullable_codes: Some(array.codes().dtype().is_nullable()), + all_values_referenced: Some(array.all_values_referenced), + } + .encode_to_vec(), + )) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - let metadata = ::deserialize(bytes)?; - Ok(ProstMetadata(metadata)) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], + _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = DictMetadata::decode(metadata)?; if children.len() != 2 { vortex_bail!( "Expected 2 children for dict encoding, found {}", @@ -197,7 +178,7 @@ impl VTable for Dict { // Also not the check to do here it take value validity using code validity, but this approx // is correct. if array.codes().all_invalid()? { - return Ok(ExecutionResult::done(ConstantData::new( + return Ok(ExecutionResult::done(ConstantArray::new( Scalar::null(array.dtype().as_nullable()), array.codes().len(), ))); @@ -205,7 +186,7 @@ impl VTable for Dict { let array = require_child!(array, array.values(), 1 => AnyCanonical); - let DictArrayParts { codes, values, .. } = array.into_data().into_parts(); + let DictDataParts { codes, values, .. } = array.into_data().into_parts(); let codes = codes .try_into::() diff --git a/vortex-array/src/arrays/dict/vtable/validity.rs b/vortex-array/src/arrays/dict/vtable/validity.rs index cdaeb4a08d5..fc13dcdc84d 100644 --- a/vortex-array/src/arrays/dict/vtable/validity.rs +++ b/vortex-array/src/arrays/dict/vtable/validity.rs @@ -7,7 +7,7 @@ use super::Dict; use crate::IntoArray; use crate::array::ArrayView; use crate::array::ValidityVTable; -use crate::arrays::dict::DictData; +use crate::arrays::DictArray; use crate::builtins::ArrayBuiltins; use crate::dtype::Nullability; use crate::scalar::Scalar; @@ -32,13 +32,13 @@ impl ValidityVTable for Dict { // We know codes are all valid, so the cast is free. let codes = array.codes().cast(array.codes().dtype().as_nonnullable())?; Validity::Array( - unsafe { DictData::new_unchecked(codes, values_validity) }.into_array(), + unsafe { DictArray::new_unchecked(codes, values_validity) }.into_array(), ) } (Validity::Array(_codes_validity), Validity::Array(values_validity)) => { // Create a mask representing "is the value at codes[i] valid?" let values_valid_mask = - unsafe { DictData::new_unchecked(array.codes().clone(), values_validity) } + unsafe { DictArray::new_unchecked(array.codes().clone(), values_validity) } .into_array(); let values_valid_mask = values_valid_mask .fill_null(Scalar::bool(false, Nullability::NonNullable))?; diff --git a/vortex-array/src/arrays/extension/array.rs b/vortex-array/src/arrays/extension/array.rs index f641b1202d4..757c3042484 100644 --- a/vortex-array/src/arrays/extension/array.rs +++ b/vortex-array/src/arrays/extension/array.rs @@ -6,10 +6,10 @@ use vortex_error::VortexResult; use crate::ArrayRef; use crate::array::Array; +use crate::array::ArrayParts; use crate::arrays::Extension; use crate::dtype::DType; use crate::dtype::extension::ExtDTypeRef; -use crate::stats::ArrayStats; /// The backing storage array for this extension array. pub(super) const STORAGE_SLOT: usize = 0; @@ -57,9 +57,8 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["storage"]; #[derive(Clone, Debug)] pub struct ExtensionData { /// The storage dtype. This **must** be a [`Extension::DType`] variant. - pub(super) dtype: DType, + pub(super) ext_dtype: ExtDTypeRef, pub(super) slots: Vec>, - pub(super) stats_set: ArrayStats, } impl ExtensionData { @@ -110,9 +109,8 @@ impl ExtensionData { ); Self { - dtype: DType::Extension(ext_dtype), + ext_dtype, slots: vec![Some(storage_array)], - stats_set: ArrayStats::default(), } } @@ -122,8 +120,8 @@ impl ExtensionData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + DType::Extension(self.ext_dtype.clone()) } /// Returns `true` if this array is empty. @@ -133,11 +131,7 @@ impl ExtensionData { /// The extension dtype of this array. pub fn ext_dtype(&self) -> &ExtDTypeRef { - let DType::Extension(ext) = &self.dtype else { - unreachable!("ExtensionArray: dtype must be an ExtDType") - }; - - ext + &self.ext_dtype } pub fn storage_array(&self) -> &ArrayRef { @@ -154,12 +148,17 @@ impl Array { /// /// Panics if the storage array is not compatible with the extension dtype. pub fn new(ext_dtype: ExtDTypeRef, storage_array: ArrayRef) -> Self { - Array::try_from_data(ExtensionData::new(ext_dtype, storage_array)) - .vortex_expect("ExtensionData is always valid") + let dtype = DType::Extension(ext_dtype.clone()); + let len = storage_array.len(); + let data = ExtensionData::new(ext_dtype, storage_array); + unsafe { Array::from_parts_unchecked(ArrayParts::new(Extension, dtype, len, data)) } } /// Tries to construct a new `ExtensionArray`. pub fn try_new(ext_dtype: ExtDTypeRef, storage_array: ArrayRef) -> VortexResult { - Array::try_from_data(ExtensionData::try_new(ext_dtype, storage_array)?) + let dtype = DType::Extension(ext_dtype.clone()); + let len = storage_array.len(); + let data = ExtensionData::try_new(ext_dtype, storage_array)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Extension, dtype, len, data)) }) } } diff --git a/vortex-array/src/arrays/extension/compute/cast.rs b/vortex-array/src/arrays/extension/compute/cast.rs index 161592ad264..31ceb8ccf8f 100644 --- a/vortex-array/src/arrays/extension/compute/cast.rs +++ b/vortex-array/src/arrays/extension/compute/cast.rs @@ -79,7 +79,7 @@ mod tests { let storage = Buffer::::empty().into_array(); let arr = ExtensionArray::new(ext_dtype.clone(), storage); - assert!(!arr.dtype.is_nullable()); + assert!(!arr.dtype().is_nullable()); let new_dtype = DType::Extension(ext_dtype).with_nullability(Nullability::Nullable); diff --git a/vortex-array/src/arrays/extension/vtable/mod.rs b/vortex-array/src/arrays/extension/vtable/mod.rs index 7eb48c10228..3ef3bc05617 100644 --- a/vortex-array/src/arrays/extension/vtable/mod.rs +++ b/vortex-array/src/arrays/extension/vtable/mod.rs @@ -13,7 +13,6 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; -use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; @@ -31,7 +30,6 @@ use crate::dtype::DType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::vtable; vtable!(Extension, Extension, ExtensionData); @@ -39,30 +37,13 @@ vtable!(Extension, Extension, ExtensionData); impl VTable for Extension { type ArrayData = ExtensionData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Extension - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ExtensionData) -> usize { - array.storage_array().len() - } - - fn dtype(array: &ExtensionData) -> &DType { - &array.dtype - } - - fn stats(array: &ExtensionData) -> &ArrayStats { - &array.stats_set - } - fn array_hash( array: &ExtensionData, state: &mut H, @@ -97,31 +78,45 @@ impl VTable for Extension { SLOT_NAMES[idx].to_string() } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } - fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) + fn validate(&self, data: &ExtensionData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.len() == len, + "ExtensionArray length {} does not match outer length {}", + data.len(), + len + ); + + let actual_dtype = data.dtype(); + vortex_ensure!( + &actual_dtype == dtype, + "ExtensionArray dtype {} does not match outer dtype {}", + actual_dtype, + dtype + ); + + Ok(()) } - fn build( + fn deserialize( + &self, dtype: &DType, len: usize, - _metadata: &Self::Metadata, + metadata: &[u8], + _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + if !metadata.is_empty() { + vortex_bail!( + "ExtensionArray expects empty metadata, got {} bytes", + metadata.len() + ); + } let DType::Extension(ext_dtype) = dtype else { vortex_bail!("Not an extension DType"); }; diff --git a/vortex-array/src/arrays/filter/array.rs b/vortex-array/src/arrays/filter/array.rs index 04f4312ca1f..7a0677d5d92 100644 --- a/vortex-array/src/arrays/filter/array.rs +++ b/vortex-array/src/arrays/filter/array.rs @@ -8,23 +8,15 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::array::Array; +use crate::array::ArrayParts; use crate::arrays::Filter; use crate::dtype::DType; -use crate::stats::ArrayStats; /// The source array being filtered. pub(super) const CHILD_SLOT: usize = 0; pub(super) const NUM_SLOTS: usize = 1; pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child"]; -/// Decomposed parts of the filter array. -pub struct FilterArrayParts { - /// Child array that is filtered by the mask - pub child: ArrayRef, - /// Mask to apply at filter time. Child elements with set indices are kept, the rest discarded. - pub mask: Mask, -} - // TODO(connor): Write docs on why we have this, and what we had in the old world so that the future // does not repeat the mistakes of the past. /// A lazy array that represents filtering a child array by a boolean [`Mask`]. @@ -36,9 +28,11 @@ pub struct FilterData { /// The boolean mask selecting which elements to keep. pub(super) mask: Mask, +} - /// The stats for this array. - pub(super) stats: ArrayStats, +pub struct FilterDataParts { + pub child: ArrayRef, + pub mask: Mask, } impl FilterData { @@ -46,7 +40,7 @@ impl FilterData { Self::try_new(array, mask).vortex_expect("FilterArray construction failed") } - pub fn try_new(array: ArrayRef, mask: Mask) -> VortexResult { + fn try_new(array: ArrayRef, mask: Mask) -> VortexResult { vortex_ensure_eq!( array.len(), mask.len(), @@ -58,7 +52,6 @@ impl FilterData { Ok(Self { slots: vec![Some(array)], mask, - stats: ArrayStats::default(), }) } @@ -88,29 +81,31 @@ impl FilterData { pub fn filter_mask(&self) -> &Mask { &self.mask } + + pub fn into_parts(mut self) -> FilterDataParts { + FilterDataParts { + child: self.slots[CHILD_SLOT] + .take() + .vortex_expect("FilterArray child slot"), + mask: self.mask, + } + } } impl Array { /// Creates a new `FilterArray`. pub fn new(array: ArrayRef, mask: Mask) -> Self { - Array::try_from_data(FilterData::new(array, mask)) - .vortex_expect("FilterData is always valid") + let dtype = array.dtype().clone(); + let len = mask.true_count(); + let data = FilterData::new(array, mask); + unsafe { Array::from_parts_unchecked(ArrayParts::new(Filter, dtype, len, data)) } } /// Constructs a new `FilterArray`. pub fn try_new(array: ArrayRef, mask: Mask) -> VortexResult { - Array::try_from_data(FilterData::try_new(array, mask)?) - } -} - -impl FilterData { - /// Consume the array and return its individual components. - pub fn into_parts(mut self) -> FilterArrayParts { - FilterArrayParts { - child: self.slots[CHILD_SLOT] - .take() - .vortex_expect("FilterArray child slot"), - mask: self.mask, - } + let dtype = array.dtype().clone(); + let len = mask.true_count(); + let data = FilterData::try_new(array, mask)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Filter, dtype, len, data)) }) } } diff --git a/vortex-array/src/arrays/filter/execute/struct_.rs b/vortex-array/src/arrays/filter/execute/struct_.rs index 74540a8739d..e424dbd2d33 100644 --- a/vortex-array/src/arrays/filter/execute/struct_.rs +++ b/vortex-array/src/arrays/filter/execute/struct_.rs @@ -29,13 +29,8 @@ pub fn filter_struct(array: &StructArray, mask: &Arc) -> StructArray .map(|a| a.len()) .unwrap_or_else(|| mask.true_count()); - StructArray::try_new_with_dtype( - fields, - array.struct_fields().clone(), - length, - filtered_validity, - ) - .vortex_expect("filtered StructArray fields have consistent lengths") + StructArray::try_new_with_dtype(fields, array.struct_fields(), length, filtered_validity) + .vortex_expect("filtered StructArray fields have consistent lengths") } #[cfg(test)] diff --git a/vortex-array/src/arrays/filter/mod.rs b/vortex-array/src/arrays/filter/mod.rs index d10cae11e41..7491899230e 100644 --- a/vortex-array/src/arrays/filter/mod.rs +++ b/vortex-array/src/arrays/filter/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::FilterArrayParts; pub use array::FilterData; +pub use array::FilterDataParts; pub use vtable::FilterArray; mod execute; diff --git a/vortex-array/src/arrays/filter/rules.rs b/vortex-array/src/arrays/filter/rules.rs index 402a7b14585..0095d47b422 100644 --- a/vortex-array/src/arrays/filter/rules.rs +++ b/vortex-array/src/arrays/filter/rules.rs @@ -11,7 +11,7 @@ use crate::array::ArrayView; use crate::arrays::Filter; use crate::arrays::Struct; use crate::arrays::StructArray; -use crate::arrays::struct_::StructArrayParts; +use crate::arrays::struct_::StructDataParts; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ArrayReduceRule; use crate::optimizer::rules::ParentRuleSet; @@ -69,7 +69,7 @@ impl ArrayReduceRule for FilterStructRule { }; let len = mask.true_count(); - let StructArrayParts { + let StructDataParts { fields, struct_fields, validity, diff --git a/vortex-array/src/arrays/filter/vtable.rs b/vortex-array/src/arrays/filter/vtable.rs index 42448d7ddd9..fb7bdbf9d0b 100644 --- a/vortex-array/src/arrays/filter/vtable.rs +++ b/vortex-array/src/arrays/filter/vtable.rs @@ -1,8 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::fmt::Debug; -use std::fmt::Formatter; use std::hash::Hasher; use vortex_error::VortexResult; @@ -36,7 +34,6 @@ use crate::executor::ExecutionCtx; use crate::executor::ExecutionResult; use crate::scalar::Scalar; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; @@ -51,27 +48,33 @@ impl Filter { impl VTable for Filter { type ArrayData = FilterData; - type Metadata = FilterMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &FilterData) -> &Self { - &Filter - } fn id(&self) -> ArrayId { Self::ID } - fn len(array: &FilterData) -> usize { - array.mask.true_count() - } - - fn dtype(array: &FilterData) -> &DType { - array.child().dtype() - } - - fn stats(array: &FilterData) -> &ArrayStats { - &array.stats + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.child().dtype() == dtype, + "FilterArray dtype {} does not match outer dtype {}", + data.child().dtype(), + dtype + ); + vortex_ensure!( + data.len() == len, + "FilterArray length {} does not match outer length {}", + data.len(), + len + ); + vortex_ensure!( + data.child().len() == data.mask.len(), + "FilterArray child length {} does not match mask length {}", + data.child().len(), + data.mask.len() + ); + Ok(()) } fn array_hash(array: &FilterData, state: &mut H, precision: Precision) { @@ -104,35 +107,22 @@ impl VTable for Filter { SLOT_NAMES[idx].to_string() } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(FilterMetadata(array.mask.clone())) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { // TODO(joe): make this configurable vortex_bail!("Filter array is not serializable") } fn deserialize( - _bytes: &[u8], + &self, _dtype: &DType, _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - vortex_bail!("Filter array is not serializable") - } + _metadata: &[u8], - fn build( - dtype: &DType, - len: usize, - metadata: &FilterMetadata, _buffers: &[BufferHandle], - children: &dyn ArrayChildren, + _children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { - assert_eq!(len, metadata.0.true_count()); - let child = children.get(0, dtype, metadata.0.len())?; - FilterData::try_new(child, metadata.0.clone()) + vortex_bail!("Filter array is not serializable") } fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { @@ -189,17 +179,3 @@ impl ValidityVTable for Filter { array.child().validity()?.filter(&array.mask) } } - -pub struct FilterMetadata(pub(super) Mask); - -impl Debug for FilterMetadata { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{} / {} => {}", - self.0.true_count(), - self.0.len(), - self.0.density() - ) - } -} diff --git a/vortex-array/src/arrays/fixed_size_list/array.rs b/vortex-array/src/arrays/fixed_size_list/array.rs index 9d5dda32f1b..eb1ef6d7aff 100644 --- a/vortex-array/src/arrays/fixed_size_list/array.rs +++ b/vortex-array/src/arrays/fixed_size_list/array.rs @@ -9,11 +9,12 @@ use vortex_error::vortex_ensure; use crate::ArrayRef; use crate::array::Array; +use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::FixedSizeList; use crate::dtype::DType; -use crate::stats::ArrayStats; +use crate::dtype::Nullability; use crate::validity::Validity; /// The `elements` data array, where each fixed-size list scalar is a _slice_ of the `elements` @@ -80,10 +81,8 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["elements", "validity"]; /// ``` #[derive(Clone, Debug)] pub struct FixedSizeListData { - /// The [`DType`] of the fixed-size list. - /// - /// This type **must** be the variant [`DType::FixedSizeList`]. - pub(super) dtype: DType, + /// The nullability of the fixed-size list array. + pub(super) nullability: Nullability, /// Slots holding [elements]. pub(super) slots: Vec>, @@ -100,10 +99,7 @@ pub struct FixedSizeListData { /// The main reason we need to store this (rather than calculate it on the fly via `list_size` /// and `elements.len()`) is because in the degenerate case where `list_size == 0`, we cannot /// use `0 / 0` to determine the length. - pub(super) len: usize, - - /// The stats for this array. - pub(super) stats_set: ArrayStats, + pub(super) degenerate_len: usize, } impl FixedSizeListData { @@ -126,7 +122,7 @@ impl FixedSizeListData { /// /// Returns an error if the provided components do not satisfy the invariants documented /// in `FixedSizeListArray::new_unchecked`. - pub fn try_new( + pub(crate) fn try_new( elements: ArrayRef, list_size: u32, validity: Validity, @@ -168,22 +164,22 @@ impl FixedSizeListData { let validity_slot = validity_to_child(&validity, len); Self { - dtype: DType::FixedSizeList(Arc::new(elements.dtype().clone()), list_size, nullability), + nullability, slots: vec![Some(elements), validity_slot], list_size, - len, - stats_set: Default::default(), + degenerate_len: if list_size == 0 { len } else { 0 }, } } pub fn into_parts(mut self) -> (ArrayRef, Validity, DType) { + let dtype = self.dtype(); let validity = self.validity(); ( self.slots[ELEMENTS_SLOT] .take() .vortex_expect("FixedSizeListArray elements slot"), validity, - self.dtype, + dtype, ) } @@ -224,13 +220,21 @@ impl FixedSizeListData { } /// Returns the dtype of the array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + DType::FixedSizeList( + Arc::new(self.elements().dtype().clone()), + self.list_size, + self.nullability, + ) } /// Returns the length of the array. pub fn len(&self) -> usize { - self.len + if self.list_size == 0 { + self.degenerate_len + } else { + self.elements().len() / self.list_size as usize + } } /// Returns `true` if the array is empty. @@ -241,7 +245,7 @@ impl FixedSizeListData { /// Returns the validity of the array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) } /// Returns the validity as a [`Mask`](vortex_mask::Mask). @@ -265,8 +269,13 @@ impl FixedSizeListData { impl Array { /// Creates a new `FixedSizeListArray`. pub fn new(elements: ArrayRef, list_size: u32, validity: Validity, len: usize) -> Self { - Array::try_from_data(FixedSizeListData::new(elements, list_size, validity, len)) - .vortex_expect("FixedSizeListData is always valid") + let dtype = DType::FixedSizeList( + Arc::new(elements.dtype().clone()), + list_size, + validity.nullability(), + ); + let data = FixedSizeListData::new(elements, list_size, validity, len); + unsafe { Array::from_parts_unchecked(ArrayParts::new(FixedSizeList, dtype, len, data)) } } /// Constructs a new `FixedSizeListArray`. @@ -276,9 +285,17 @@ impl Array { validity: Validity, len: usize, ) -> VortexResult { - Array::try_from_data(FixedSizeListData::try_new( - elements, list_size, validity, len, - )?) + let dtype = DType::FixedSizeList( + Arc::new(elements.dtype().clone()), + list_size, + validity.nullability(), + ); + let data = FixedSizeListData::try_new(elements, list_size, validity, len)?; + Ok( + unsafe { + Array::from_parts_unchecked(ArrayParts::new(FixedSizeList, dtype, len, data)) + }, + ) } /// Creates a new `FixedSizeListArray` without validation. @@ -292,20 +309,23 @@ impl Array { validity: Validity, len: usize, ) -> Self { - Array::try_from_data(unsafe { - FixedSizeListData::new_unchecked(elements, list_size, validity, len) - }) - .vortex_expect("FixedSizeListData is always valid") + let dtype = DType::FixedSizeList( + Arc::new(elements.dtype().clone()), + list_size, + validity.nullability(), + ); + let data = unsafe { FixedSizeListData::new_unchecked(elements, list_size, validity, len) }; + unsafe { Array::from_parts_unchecked(ArrayParts::new(FixedSizeList, dtype, len, data)) } } } impl FixedSizeListData { pub fn fixed_size_list_elements_at(&self, index: usize) -> VortexResult { debug_assert!( - index < self.len, + index < self.len(), "index {} out of bounds: the len is {}", index, - self.len, + self.len(), ); debug_assert!(self.validity().is_valid(index).unwrap_or(false)); diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs index 31fa5342185..ab3e468c187 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs @@ -10,7 +10,6 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; -use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; @@ -27,7 +26,6 @@ use crate::dtype::DType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; mod kernel; @@ -46,29 +44,13 @@ impl FixedSizeList { impl VTable for FixedSizeList { type ArrayData = FixedSizeListData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &FixedSizeListData) -> &Self { - &FixedSizeList - } fn id(&self) -> ArrayId { Self::ID } - fn len(array: &FixedSizeListData) -> usize { - array.len - } - - fn dtype(array: &FixedSizeListData) -> &DType { - &array.dtype - } - - fn stats(array: &FixedSizeListData) -> &ArrayStats { - &array.stats_set - } - fn array_hash( array: &FixedSizeListData, state: &mut H, @@ -118,34 +100,45 @@ impl VTable for FixedSizeList { Self::PARENT_KERNELS.execute(array, parent, child_idx, ctx) } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } - fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) + fn validate(&self, data: &FixedSizeListData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.len() == len, + "FixedSizeListArray length {} does not match outer length {}", + data.len(), + len + ); + + let actual_dtype = data.dtype(); + vortex_ensure!( + &actual_dtype == dtype, + "FixedSizeListArray dtype {} does not match outer dtype {}", + actual_dtype, + dtype + ); + + Ok(()) } - /// Builds a [`FixedSizeListArray`]. - /// - /// This method expects 1 or 2 children (a second child indicates a validity array). - fn build( + fn deserialize( + &self, dtype: &DType, len: usize, - _metadata: &Self::Metadata, + metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + if !metadata.is_empty() { + vortex_bail!( + "FixedSizeListArray expects empty metadata, got {} bytes", + metadata.len() + ); + } vortex_ensure!( buffers.is_empty(), "`FixedSizeList::build` expects no buffers" diff --git a/vortex-array/src/arrays/list/array.rs b/vortex-array/src/arrays/list/array.rs index 2678ab551e2..b8f99c35804 100644 --- a/vortex-array/src/arrays/list/array.rs +++ b/vortex-array/src/arrays/list/array.rs @@ -16,6 +16,7 @@ use crate::LEGACY_SESSION; use crate::VortexSessionExecute; use crate::aggregate_fn::fns::min_max::min_max; use crate::array::Array; +use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::ConstantArray; @@ -24,10 +25,10 @@ use crate::arrays::Primitive; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::dtype::NativePType; +use crate::dtype::Nullability; use crate::match_each_integer_ptype; use crate::match_each_native_ptype; use crate::scalar_fn::fns::operators::Operator; -use crate::stats::ArrayStats; use crate::validity::Validity; /// The elements data array containing all list elements concatenated together. @@ -93,16 +94,15 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["elements", "offsets", "validi /// ``` #[derive(Clone, Debug)] pub struct ListData { - pub(super) dtype: DType, + pub(super) nullability: Nullability, pub(super) slots: Vec>, - pub(super) stats_set: ArrayStats, } -pub struct ListArrayParts { - pub dtype: DType, +pub struct ListDataParts { pub elements: ArrayRef, pub offsets: ArrayRef, pub validity: Validity, + pub dtype: DType, } impl ListData { @@ -124,7 +124,7 @@ impl ListData { /// /// Returns an error if the provided components do not satisfy the invariants documented in /// `ListArray::new_unchecked`. - pub fn try_new( + pub(crate) fn try_new( elements: ArrayRef, offsets: ArrayRef, validity: Validity, @@ -160,9 +160,8 @@ impl ListData { let validity_slot = validity_to_child(&validity, len); Self { - dtype: DType::List(Arc::new(elements.dtype().clone()), validity.nullability()), + nullability: validity.nullability(), slots: vec![Some(elements), Some(offsets), validity_slot], - stats_set: Default::default(), } } @@ -250,25 +249,9 @@ impl ListData { Ok(()) } - - /// Splits an array into its parts - pub fn into_parts(mut self) -> ListArrayParts { - let validity = self.validity(); - ListArrayParts { - dtype: self.dtype, - elements: self.slots[ELEMENTS_SLOT] - .take() - .vortex_expect("ListArray elements slot"), - offsets: self.slots[OFFSETS_SLOT] - .take() - .vortex_expect("ListArray offsets slot"), - validity, - } - } - /// Returns the dtype of the array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + DType::List(Arc::new(self.elements().dtype().clone()), self.nullability) } /// Returns the length of the array. @@ -284,7 +267,7 @@ impl ListData { /// Returns the validity of the array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) } /// Returns the validity as a [`Mask`](vortex_mask::Mask). @@ -340,11 +323,8 @@ impl ListData { } /// Returns the element dtype of the list array. - pub fn element_dtype(&self) -> &Arc { - match &self.dtype { - DType::List(element_dtype, _) => element_dtype, - _ => vortex_panic!("ListArray has invalid dtype {}", self.dtype), - } + pub fn element_dtype(&self) -> &DType { + self.elements().dtype() } /// Returns the elements array. @@ -354,6 +334,21 @@ impl ListData { .vortex_expect("ListArray elements slot") } + pub fn into_parts(mut self) -> ListDataParts { + let dtype = self.dtype(); + let validity = self.validity(); + ListDataParts { + elements: self.slots[ELEMENTS_SLOT] + .take() + .vortex_expect("ListArray elements slot"), + offsets: self.slots[OFFSETS_SLOT] + .take() + .vortex_expect("ListArray offsets slot"), + validity, + dtype, + } + } + // TODO(connor)[ListView]: Create 2 functions `reset_offsets` and `recursive_reset_offsets`, // where `reset_offsets` is infallible. // Also, `reset_offsets` can be made more efficient by replacing `sub_scalar` with a match on @@ -363,8 +358,10 @@ impl ListData { impl Array { /// Creates a new `ListArray`. pub fn new(elements: ArrayRef, offsets: ArrayRef, validity: Validity) -> Self { - Array::try_from_data(ListData::new(elements, offsets, validity)) - .vortex_expect("ListData is always valid") + let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); + let len = offsets.len().saturating_sub(1); + let data = ListData::new(elements, offsets, validity); + unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) } } /// Constructs a new `ListArray`. @@ -373,7 +370,10 @@ impl Array { offsets: ArrayRef, validity: Validity, ) -> VortexResult { - Array::try_from_data(ListData::try_new(elements, offsets, validity)?) + let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); + let len = offsets.len().saturating_sub(1); + let data = ListData::try_new(elements, offsets, validity)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) }) } /// Creates a new `ListArray` without validation. @@ -382,8 +382,10 @@ impl Array { /// /// See [`ListData::new_unchecked`]. pub unsafe fn new_unchecked(elements: ArrayRef, offsets: ArrayRef, validity: Validity) -> Self { - Array::try_from_data(unsafe { ListData::new_unchecked(elements, offsets, validity) }) - .vortex_expect("ListData is always valid") + let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); + let len = offsets.len().saturating_sub(1); + let data = unsafe { ListData::new_unchecked(elements, offsets, validity) }; + unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) } } } @@ -393,7 +395,12 @@ impl ListData { if recurse && elements.is_canonical() { elements = elements.to_canonical()?.compact()?.into_array(); } else if recurse && let Some(child_list_array) = elements.as_opt::() { - elements = child_list_array.reset_offsets(recurse)?.into_array(); + let data = child_list_array.reset_offsets(recurse)?; + let dtype = data.dtype(); + let len = data.len(); + elements = + unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) } + .into_array(); } let offsets = self.offsets(); diff --git a/vortex-array/src/arrays/list/mod.rs b/vortex-array/src/arrays/list/mod.rs index 753a2a35758..3147097ca80 100644 --- a/vortex-array/src/arrays/list/mod.rs +++ b/vortex-array/src/arrays/list/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::ListArrayParts; pub use array::ListData; +pub use array::ListDataParts; pub use vtable::ListArray; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/list/tests.rs b/vortex-array/src/arrays/list/tests.rs index 874be33275b..f815acb3bb0 100644 --- a/vortex-array/src/arrays/list/tests.rs +++ b/vortex-array/src/arrays/list/tests.rs @@ -14,6 +14,8 @@ use crate::Canonical; use crate::IntoArray; use crate::LEGACY_SESSION; use crate::VortexSessionExecute; +use crate::array::Array; +use crate::array::ArrayParts; use crate::arrays::FilterArray; use crate::arrays::List; use crate::arrays::PrimitiveArray; @@ -913,9 +915,23 @@ fn test_recursive_compact_list_of_lists() { assert_eq!(recursive_flat_elements.len(), 7); // Verify data integrity is preserved + let non_recursive_array = Array::try_from_parts(ArrayParts::new( + List, + non_recursive.dtype(), + non_recursive.len(), + non_recursive.clone(), + )) + .unwrap(); + let recursive_array = Array::try_from_parts(ArrayParts::new( + List, + recursive.dtype(), + recursive.len(), + recursive.clone(), + )) + .unwrap(); assert_eq!( - non_recursive.clone().into_array().scalar_at(0).unwrap(), - recursive.clone().into_array().scalar_at(0).unwrap() + non_recursive_array.scalar_at(0).unwrap(), + recursive_array.scalar_at(0).unwrap() ); } diff --git a/vortex-array/src/arrays/list/vtable/mod.rs b/vortex-array/src/arrays/list/vtable/mod.rs index 051ee80d2d0..dd999ee3419 100644 --- a/vortex-array/src/arrays/list/vtable/mod.rs +++ b/vortex-array/src/arrays/list/vtable/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use prost::Message; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -12,7 +13,6 @@ use crate::ExecutionCtx; use crate::ExecutionResult; use crate::IntoArray; use crate::Precision; -use crate::ProstMetadata; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; @@ -29,10 +29,7 @@ use crate::dtype::Nullability; use crate::dtype::PType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; -use crate::metadata::DeserializeMetadata; -use crate::metadata::SerializeMetadata; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; mod operations; @@ -50,29 +47,13 @@ pub struct ListMetadata { impl VTable for List { type ArrayData = ListData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &ListData) -> &Self { - &List - } fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ListData) -> usize { - array.offsets().len().saturating_sub(1) - } - - fn dtype(array: &ListData) -> &DType { - &array.dtype - } - - fn stats(array: &ListData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &ListData, state: &mut H, precision: Precision) { array.elements().array_hash(state, precision); array.offsets().array_hash(state, precision); @@ -105,36 +86,46 @@ impl VTable for List { PARENT_RULES.evaluate(array, parent, child_idx) } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(ListMetadata { - elements_len: array.elements().len() as u64, - offset_ptype: PType::try_from(array.offsets().dtype())? as i32, - })) + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some( + ListMetadata { + elements_len: array.elements().len() as u64, + offset_ptype: PType::try_from(array.offsets().dtype())? as i32, + } + .encode_to_vec(), + )) } - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(SerializeMetadata::serialize(metadata))) - } + fn validate(&self, data: &ListData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.len() == len, + "ListArray length {} does not match outer length {}", + data.len(), + len + ); - fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata( - as DeserializeMetadata>::deserialize(bytes)?, - )) + let actual_dtype = data.dtype(); + vortex_ensure!( + &actual_dtype == dtype, + "ListArray dtype {} does not match outer dtype {}", + actual_dtype, + dtype + ); + + Ok(()) } - fn build( + fn deserialize( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], + _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = ListMetadata::decode(metadata)?; let validity = if children.len() == 2 { Validity::from(dtype.nullability()) } else if children.len() == 3 { @@ -150,12 +141,12 @@ impl VTable for List { let elements = children.get( 0, element_dtype.as_ref(), - usize::try_from(metadata.0.elements_len)?, + usize::try_from(metadata.elements_len)?, )?; let offsets = children.get( 1, - &DType::Primitive(metadata.0.offset_ptype(), Nullability::NonNullable), + &DType::Primitive(metadata.offset_ptype(), Nullability::NonNullable), len + 1, )?; diff --git a/vortex-array/src/arrays/listview/array.rs b/vortex-array/src/arrays/listview/array.rs index 7a8c7122d4c..c25dce40f43 100644 --- a/vortex-array/src/arrays/listview/array.rs +++ b/vortex-array/src/arrays/listview/array.rs @@ -13,6 +13,7 @@ use vortex_error::vortex_err; use crate::ArrayRef; use crate::ToCanonical; use crate::array::Array; +use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::ListView; @@ -21,8 +22,8 @@ use crate::arrays::PrimitiveArray; use crate::arrays::bool; use crate::dtype::DType; use crate::dtype::IntegerPType; +use crate::dtype::Nullability; use crate::match_each_integer_ptype; -use crate::stats::ArrayStats; use crate::validity::Validity; /// The `elements` data array, where each list scalar is a _slice_ of the `elements` array, and @@ -107,10 +108,8 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["elements", "offsets", "sizes" /// [`ListArray`]: crate::arrays::ListArray #[derive(Clone, Debug)] pub struct ListViewData { - /// The [`DType`] of the list array. - /// - /// This type **must** be the variant [`DType::List`]. - pub(super) dtype: DType, + /// The nullability of the list array. + pub(super) nullability: Nullability, /// Slots holding [elements, offsets, sizes]. pub(super) slots: Vec>, @@ -124,12 +123,9 @@ pub struct ListViewData { /// `offsets[i] + sizes[i]` are in order), conversions can bypass the very expensive rebuild /// process which must rebuild the array from scratch. is_zero_copy_to_list: bool, - - /// The stats for this array. - pub(super) stats_set: ArrayStats, } -pub struct ListViewArrayParts { +pub struct ListViewDataParts { pub elements_dtype: Arc, /// See `ListViewArray::elements` @@ -175,10 +171,9 @@ impl ListViewData { let validity_slot = validity_to_child(&validity, len); Ok(Self { - dtype: DType::List(Arc::new(elements.dtype().clone()), validity.nullability()), + nullability: validity.nullability(), slots: vec![Some(elements), Some(offsets), Some(sizes), validity_slot], is_zero_copy_to_list: false, - stats_set: Default::default(), }) } @@ -216,10 +211,9 @@ impl ListViewData { let validity_slot = validity_to_child(&validity, len); Self { - dtype: DType::List(Arc::new(elements.dtype().clone()), validity.nullability()), + nullability: validity.nullability(), slots: vec![Some(elements), Some(offsets), Some(sizes), validity_slot], is_zero_copy_to_list: false, - stats_set: Default::default(), } } @@ -343,11 +337,10 @@ impl ListViewData { .is_ok() } - pub fn into_parts(mut self) -> ListViewArrayParts { + pub fn into_parts(mut self) -> ListViewDataParts { let validity = self.validity(); - let dtype = self.dtype.into_list_element_opt().vortex_expect("is list"); - ListViewArrayParts { - elements_dtype: dtype, + ListViewDataParts { + elements_dtype: Arc::new(self.elements().dtype().clone()), elements: self.slots[ELEMENTS_SLOT] .take() .vortex_expect("ListViewArray elements slot"), @@ -362,8 +355,8 @@ impl ListViewData { } /// Returns the dtype of the array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + DType::List(Arc::new(self.elements().dtype().clone()), self.nullability) } /// Returns the length of the array. @@ -380,7 +373,7 @@ impl ListViewData { /// Returns the validity of the array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) } /// Returns the validity as a [`Mask`](vortex_mask::Mask). @@ -485,8 +478,10 @@ impl ListViewData { impl Array { /// Creates a new `ListViewArray`. pub fn new(elements: ArrayRef, offsets: ArrayRef, sizes: ArrayRef, validity: Validity) -> Self { - Array::try_from_data(ListViewData::new(elements, offsets, sizes, validity)) - .vortex_expect("ListViewData is always valid") + let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); + let len = offsets.len(); + let data = ListViewData::new(elements, offsets, sizes, validity); + unsafe { Array::from_parts_unchecked(ArrayParts::new(ListView, dtype, len, data)) } } /// Constructs a new `ListViewArray`. @@ -496,7 +491,10 @@ impl Array { sizes: ArrayRef, validity: Validity, ) -> VortexResult { - Array::try_from_data(ListViewData::try_new(elements, offsets, sizes, validity)?) + let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); + let len = offsets.len(); + let data = ListViewData::try_new(elements, offsets, sizes, validity)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(ListView, dtype, len, data)) }) } /// Creates a new `ListViewArray` without validation. @@ -510,10 +508,10 @@ impl Array { sizes: ArrayRef, validity: Validity, ) -> Self { - Array::try_from_data(unsafe { - ListViewData::new_unchecked(elements, offsets, sizes, validity) - }) - .vortex_expect("ListViewData is always valid") + let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); + let len = offsets.len(); + let data = unsafe { ListViewData::new_unchecked(elements, offsets, sizes, validity) }; + unsafe { Array::from_parts_unchecked(ArrayParts::new(ListView, dtype, len, data)) } } /// Mark whether this list view can be zero-copy converted to a list. @@ -522,8 +520,10 @@ impl Array { /// /// See [`ListViewData::with_zero_copy_to_list`]. pub unsafe fn with_zero_copy_to_list(self, is_zctl: bool) -> Self { - Array::try_from_data(unsafe { self.into_data().with_zero_copy_to_list(is_zctl) }) - .vortex_expect("data is always valid") + let dtype = self.dtype().clone(); + let len = self.len(); + let data = unsafe { self.into_data().with_zero_copy_to_list(is_zctl) }; + unsafe { Array::from_parts_unchecked(ArrayParts::new(ListView, dtype, len, data)) } } } diff --git a/vortex-array/src/arrays/listview/conversion.rs b/vortex-array/src/arrays/listview/conversion.rs index be4ac64ab41..d2334c59526 100644 --- a/vortex-array/src/arrays/listview/conversion.rs +++ b/vortex-array/src/arrays/listview/conversion.rs @@ -10,8 +10,10 @@ use crate::ExecutionCtx; use crate::IntoArray; use crate::ToCanonical; use crate::array::Array; +use crate::array::ArrayParts; use crate::arrays::ExtensionArray; use crate::arrays::FixedSizeListArray; +use crate::arrays::List; use crate::arrays::ListArray; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; @@ -36,8 +38,11 @@ pub fn list_view_from_list(list: ListArray, ctx: &mut ExecutionCtx) -> VortexRes // We reset the offsets here because mostly for convenience, and also because callers of this // function might not expect the output `ListViewArray` to have a bunch of leading and trailing // garbage data when they turn it back into a `ListArray`. + let data = list.reset_offsets(false).vortex_expect("This can't fail"); + let dtype = data.dtype(); + let len = data.len(); let list: ListArray = - Array::try_from_data(list.reset_offsets(false).vortex_expect("This can't fail"))?; + unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) }; let list_offsets = list.offsets().clone(); diff --git a/vortex-array/src/arrays/listview/mod.rs b/vortex-array/src/arrays/listview/mod.rs index 4ca62ef0820..20dd6fe15d5 100644 --- a/vortex-array/src/arrays/listview/mod.rs +++ b/vortex-array/src/arrays/listview/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::ListViewArrayParts; pub use array::ListViewData; +pub use array::ListViewDataParts; pub use vtable::ListViewArray; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/listview/vtable/mod.rs b/vortex-array/src/arrays/listview/vtable/mod.rs index e8fab2f9d82..9dae3483a16 100644 --- a/vortex-array/src/arrays/listview/vtable/mod.rs +++ b/vortex-array/src/arrays/listview/vtable/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use prost::Message; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -8,12 +9,9 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; -use crate::DeserializeMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; -use crate::ProstMetadata; -use crate::SerializeMetadata; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; @@ -29,7 +27,6 @@ use crate::dtype::PType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; mod operations; @@ -56,30 +53,13 @@ pub struct ListViewMetadata { impl VTable for ListView { type ArrayData = ListViewData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &ListViewData) -> &Self { - &ListView - } fn id(&self) -> ArrayId { Self::ID } - fn len(array: &ListViewData) -> usize { - debug_assert_eq!(array.offsets().len(), array.sizes().len()); - array.offsets().len() - } - - fn dtype(array: &ListViewData) -> &DType { - &array.dtype - } - - fn stats(array: &ListViewData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &ListViewData, state: &mut H, precision: Precision) { array.elements().array_hash(state, precision); array.offsets().array_hash(state, precision); @@ -106,36 +86,47 @@ impl VTable for ListView { vortex_panic!("ListViewArray buffer_name index {idx} out of bounds") } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(ListViewMetadata { - elements_len: array.elements().len() as u64, - offset_ptype: PType::try_from(array.offsets().dtype())? as i32, - size_ptype: PType::try_from(array.sizes().dtype())? as i32, - })) + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some( + ListViewMetadata { + elements_len: array.elements().len() as u64, + offset_ptype: PType::try_from(array.offsets().dtype())? as i32, + size_ptype: PType::try_from(array.sizes().dtype())? as i32, + } + .encode_to_vec(), + )) } - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) - } + fn validate(&self, data: &ListViewData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.len() == len, + "ListViewArray length {} does not match outer length {}", + data.len(), + len + ); - fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - let metadata = ::deserialize(bytes)?; - Ok(ProstMetadata(metadata)) + let actual_dtype = data.dtype(); + vortex_ensure!( + &actual_dtype == dtype, + "ListViewArray dtype {} does not match outer dtype {}", + actual_dtype, + dtype + ); + + Ok(()) } - fn build( + fn deserialize( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = ListViewMetadata::decode(metadata)?; vortex_ensure!( buffers.is_empty(), "`ListViewArray::build` expects no buffers" @@ -161,20 +152,20 @@ impl VTable for ListView { let elements = children.get( 0, element_dtype.as_ref(), - usize::try_from(metadata.0.elements_len)?, + usize::try_from(metadata.elements_len)?, )?; // Get offsets with proper type from metadata. let offsets = children.get( 1, - &DType::Primitive(metadata.0.offset_ptype(), Nullability::NonNullable), + &DType::Primitive(metadata.offset_ptype(), Nullability::NonNullable), len, )?; // Get sizes with proper type from metadata. let sizes = children.get( 2, - &DType::Primitive(metadata.0.size_ptype(), Nullability::NonNullable), + &DType::Primitive(metadata.size_ptype(), Nullability::NonNullable), len, )?; diff --git a/vortex-array/src/arrays/listview/vtable/operations.rs b/vortex-array/src/arrays/listview/vtable/operations.rs index 4a15c3dd8d0..b32bb87de1f 100644 --- a/vortex-array/src/arrays/listview/vtable/operations.rs +++ b/vortex-array/src/arrays/listview/vtable/operations.rs @@ -26,7 +26,7 @@ impl OperationsVTable for ListView { Ok(Scalar::list( Arc::new(list.dtype().clone()), children, - array.dtype.nullability(), + array.dtype().nullability(), )) } } diff --git a/vortex-array/src/arrays/masked/array.rs b/vortex-array/src/arrays/masked/array.rs index 0c2bc773817..9e011140930 100644 --- a/vortex-array/src/arrays/masked/array.rs +++ b/vortex-array/src/arrays/masked/array.rs @@ -7,11 +7,12 @@ use vortex_error::vortex_bail; use crate::ArrayRef; use crate::array::Array; +use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Masked; use crate::dtype::DType; -use crate::stats::ArrayStats; +use crate::dtype::Nullability; use crate::validity::Validity; /// The underlying child array being masked. @@ -24,12 +25,10 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child", "validity"]; #[derive(Clone, Debug)] pub struct MaskedData { pub(super) slots: Vec>, - pub(super) dtype: DType, - pub(super) stats: ArrayStats, } impl MaskedData { - pub fn try_new(child: ArrayRef, validity: Validity) -> VortexResult { + pub(crate) fn try_new(child: ArrayRef, validity: Validity) -> VortexResult { if matches!(validity, Validity::NonNullable) { vortex_bail!("MaskedArray must have nullable validity, got {validity:?}") } @@ -46,20 +45,17 @@ impl MaskedData { // MaskedArray's nullability is determined solely by its validity, not the child's dtype. // The child can have nullable dtype but must not have any actual null values. - let dtype = child.dtype().as_nullable(); let len = child.len(); let validity_slot = validity_to_child(&validity, len); Ok(Self { slots: vec![Some(child), validity_slot], - dtype, - stats: ArrayStats::default(), }) } /// Returns the dtype of the array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + self.child().dtype().as_nullable() } /// Returns the length of the array. @@ -75,7 +71,7 @@ impl MaskedData { /// Returns the validity of the array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], Nullability::Nullable) } /// Returns the validity as a [`Mask`](vortex_mask::Mask). @@ -93,6 +89,9 @@ impl MaskedData { impl Array { /// Constructs a new `MaskedArray`. pub fn try_new(child: ArrayRef, validity: Validity) -> VortexResult { - Array::try_from_data(MaskedData::try_new(child, validity)?) + let dtype = child.dtype().as_nullable(); + let len = child.len(); + let data = MaskedData::try_new(child, validity)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Masked, dtype, len, data)) }) } } diff --git a/vortex-array/src/arrays/masked/compute/filter.rs b/vortex-array/src/arrays/masked/compute/filter.rs index a96c57feb1f..0eb8fb25011 100644 --- a/vortex-array/src/arrays/masked/compute/filter.rs +++ b/vortex-array/src/arrays/masked/compute/filter.rs @@ -8,8 +8,8 @@ use crate::ArrayRef; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Masked; +use crate::arrays::MaskedArray; use crate::arrays::filter::FilterReduce; -use crate::arrays::masked::MaskedData; impl FilterReduce for Masked { fn filter(array: ArrayView<'_, Masked>, mask: &Mask) -> VortexResult> { @@ -22,7 +22,7 @@ impl FilterReduce for Masked { // Construct new MaskedArray Ok(Some( - MaskedData::try_new(filtered_child, filtered_validity)?.into_array(), + MaskedArray::try_new(filtered_child, filtered_validity)?.into_array(), )) } } diff --git a/vortex-array/src/arrays/masked/compute/slice.rs b/vortex-array/src/arrays/masked/compute/slice.rs index afa9b73a6e6..1be55ea6c30 100644 --- a/vortex-array/src/arrays/masked/compute/slice.rs +++ b/vortex-array/src/arrays/masked/compute/slice.rs @@ -9,7 +9,7 @@ use crate::ArrayRef; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Masked; -use crate::arrays::masked::MaskedData; +use crate::arrays::MaskedArray; use crate::arrays::slice::SliceReduce; impl SliceReduce for Masked { @@ -17,6 +17,6 @@ impl SliceReduce for Masked { let child = array.child().slice(range.clone())?; let validity = array.validity().slice(range)?; - Ok(Some(MaskedData::try_new(child, validity)?.into_array())) + Ok(Some(MaskedArray::try_new(child, validity)?.into_array())) } } diff --git a/vortex-array/src/arrays/masked/compute/take.rs b/vortex-array/src/arrays/masked/compute/take.rs index bd9a45b4711..7061529b897 100644 --- a/vortex-array/src/arrays/masked/compute/take.rs +++ b/vortex-array/src/arrays/masked/compute/take.rs @@ -7,8 +7,8 @@ use crate::ArrayRef; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Masked; +use crate::arrays::MaskedArray; use crate::arrays::dict::TakeReduce; -use crate::arrays::masked::MaskedData; use crate::builtins::ArrayBuiltins; use crate::scalar::Scalar; @@ -28,7 +28,7 @@ impl TakeReduce for Masked { // Construct new MaskedArray Ok(Some( - MaskedData::try_new(taken_child, taken_validity)?.into_array(), + MaskedArray::try_new(taken_child, taken_validity)?.into_array(), )) } } diff --git a/vortex-array/src/arrays/masked/execute.rs b/vortex-array/src/arrays/masked/execute.rs index bbf5b7fcb87..0f11a537ba2 100644 --- a/vortex-array/src/arrays/masked/execute.rs +++ b/vortex-array/src/arrays/masked/execute.rs @@ -177,7 +177,7 @@ fn mask_validity_struct( let len = array.len(); let new_validity = combine_validity(&array.validity(), mask, len, ctx)?; let fields = array.unmasked_fields(); - let struct_fields = array.struct_fields().clone(); + let struct_fields = array.struct_fields(); // SAFETY: We're only changing validity, not the data structure Ok(unsafe { StructArray::new_unchecked(fields, struct_fields, len, new_validity) }) } diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index b7069998f99..4a79d53b78b 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -12,14 +12,13 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::Canonical; -use crate::EmptyMetadata; use crate::IntoArray; use crate::Precision; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; use crate::array::VTable; -use crate::arrays::constant::ConstantData; +use crate::arrays::ConstantArray; use crate::arrays::masked::MaskedData; use crate::arrays::masked::array::NUM_SLOTS; use crate::arrays::masked::array::SLOT_NAMES; @@ -33,7 +32,6 @@ use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::scalar::Scalar; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; vtable!(Masked, Masked, MaskedData); @@ -48,28 +46,23 @@ impl Masked { impl VTable for Masked { type ArrayData = MaskedData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Masked - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &MaskedData) -> usize { - array.child().len() - } - - fn dtype(array: &MaskedData) -> &DType { - &array.dtype - } - - fn stats(array: &MaskedData) -> &ArrayStats { - &array.stats + fn validate(&self, data: &MaskedData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.child().len() == len, + "MaskedArray child length mismatch" + ); + vortex_ensure!( + data.dtype() == *dtype, + "MaskedArray dtype does not match child and validity" + ); + Ok(()) } fn array_hash(array: &MaskedData, state: &mut H, precision: Precision) { @@ -94,31 +87,26 @@ impl VTable for Masked { None } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) - } - - fn build( + &self, dtype: &DType, len: usize, - _metadata: &Self::Metadata, + metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + if !metadata.is_empty() { + vortex_bail!( + "MaskedArray expects empty metadata, got {} bytes", + metadata.len() + ); + } if !buffers.is_empty() { vortex_bail!("Expected 0 buffer, got {}", buffers.len()); } @@ -147,7 +135,7 @@ impl VTable for Masked { // Fast path: all masked means result is all nulls. if validity_mask.all_false() { return Ok(ExecutionResult::done( - ConstantData::new(Scalar::null(array.dtype().as_nullable()), array.len()) + ConstantArray::new(Scalar::null(array.dtype().as_nullable()), array.len()) .into_array(), )); } @@ -208,8 +196,8 @@ mod tests { use crate::arrays::MaskedArray; use crate::arrays::PrimitiveArray; use crate::dtype::Nullability; - use crate::serde::ArrayParts; use crate::serde::SerializeOptions; + use crate::serde::SerializedArray; use crate::validity::Validity; #[rstest] @@ -249,7 +237,7 @@ mod tests { } let concat = concat.freeze(); - let parts = ArrayParts::try_from(concat).unwrap(); + let parts = SerializedArray::try_from(concat).unwrap(); let decoded = parts .decode( &dtype, diff --git a/vortex-array/src/arrays/null/mod.rs b/vortex-array/src/arrays/null/mod.rs index 2643a9ae4cf..183a2008654 100644 --- a/vortex-array/src/arrays/null/mod.rs +++ b/vortex-array/src/arrays/null/mod.rs @@ -1,19 +1,18 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; -use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; use crate::array::Array; use crate::array::ArrayId; +use crate::array::ArrayParts; use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::array::VTable; @@ -23,7 +22,6 @@ use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::scalar::Scalar; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; @@ -36,28 +34,16 @@ vtable!(Null, Null, NullData); impl VTable for Null { type ArrayData = NullData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Null - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &NullData) -> usize { - array.len - } - - fn dtype(_array: &NullData) -> &DType { - &DType::Null - } - - fn stats(array: &NullData) -> &ArrayStats { - &array.stats_set + fn validate(&self, _data: &NullData, dtype: &DType, _len: usize) -> VortexResult<()> { + vortex_ensure!(*dtype == DType::Null, "NullArray dtype must be DType::Null"); + Ok(()) } fn array_hash(_array: &NullData, _state: &mut H, _precision: Precision) { @@ -99,32 +85,26 @@ impl VTable for Null { Ok(()) } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], + &self, _dtype: &DType, _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) - } + metadata: &[u8], - fn build( - _dtype: &DType, - len: usize, - _metadata: &Self::Metadata, _buffers: &[BufferHandle], _children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { - Ok(NullData::new(len)) + vortex_ensure!( + metadata.is_empty(), + "NullArray expects empty metadata, got {} bytes", + metadata.len() + ); + Ok(NullData::new()) } fn reduce_parent( @@ -169,9 +149,7 @@ impl VTable for Null { /// ``` #[derive(Clone, Debug)] pub struct NullData { - len: usize, slots: Vec>, - stats_set: ArrayStats, } #[derive(Clone, Debug)] @@ -183,32 +161,21 @@ impl Null { impl Array { pub fn new(len: usize) -> Self { - Array::try_from_data(NullData::new(len)).vortex_expect("NullData is always valid") - } -} - -impl NullData { - pub fn new(len: usize) -> Self { - Self { - len, - slots: vec![], - stats_set: Default::default(), + unsafe { + Array::from_parts_unchecked(ArrayParts::new(Null, DType::Null, len, NullData::new())) } } +} - /// Returns the dtype of the array (always [`DType::Null`]). - pub fn dtype(&self) -> &DType { - &DType::Null - } - - /// Returns the length of the array. - pub fn len(&self) -> usize { - self.len +impl Default for NullData { + fn default() -> Self { + Self::new() } +} - /// Returns `true` if the array is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 +impl NullData { + pub fn new() -> Self { + Self { slots: vec![] } } } impl OperationsVTable for Null { diff --git a/vortex-array/src/arrays/patched/array.rs b/vortex-array/src/arrays/patched/array.rs index 9c0d541f5db..4976a103dd6 100644 --- a/vortex-array/src/arrays/patched/array.rs +++ b/vortex-array/src/arrays/patched/array.rs @@ -14,6 +14,9 @@ use crate::ArrayRef; use crate::Canonical; use crate::ExecutionCtx; use crate::IntoArray; +use crate::array::Array; +use crate::array::ArrayParts; +use crate::arrays::Patched; use crate::arrays::PrimitiveArray; use crate::arrays::patched::TransposedPatches; use crate::arrays::patched::patch_lanes; @@ -24,7 +27,6 @@ use crate::dtype::PType; use crate::match_each_native_ptype; use crate::match_each_unsigned_integer_ptype; use crate::patches::Patches; -use crate::stats::ArrayStats; use crate::validity::Validity; /// An array that partially "patches" another array with new values. @@ -124,10 +126,23 @@ pub struct PatchedArray { /// should be subtracted out of the remaining offsets to get their final position in the /// executed array. pub(super) offset: usize, - /// Length of the array - pub(super) len: usize, +} - pub(super) stats_set: ArrayStats, +impl IntoArray for PatchedArray { + fn into_array(self) -> ArrayRef { + let dtype = self.base_array().dtype().clone(); + let len = self.len(); + unsafe { + Array::::from_parts_unchecked(ArrayParts::new(Patched, dtype, len, self)) + } + .into_array() + } +} + +impl From for ArrayRef { + fn from(value: PatchedArray) -> Self { + value.into_array() + } } impl PatchedArray { @@ -192,14 +207,10 @@ impl PatchedArray { ) .into_array(); - let len = inner.len(); - Ok(Self { slots: vec![Some(inner), Some(lane_offsets), Some(indices), Some(values)], n_lanes, offset: 0, - len, - stats_set: ArrayStats::default(), }) } } @@ -236,6 +247,16 @@ impl PatchedArray { .as_ref() .vortex_expect("PatchedArray values slot") } + + #[inline] + pub fn len(&self) -> usize { + self.base_array().len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } } impl PatchedArray { @@ -246,7 +267,7 @@ impl PatchedArray { /// /// Note that this function will panic if the caller requests out of bounds chunk/lane ordinals. pub(crate) fn lane_range(&self, chunk: usize, lane: usize) -> VortexResult> { - assert!(chunk * 1024 <= self.len + self.offset); + assert!(chunk * 1024 <= self.len() + self.offset); assert!(lane < self.n_lanes); let start = self.lane_offsets().scalar_at(chunk * self.n_lanes + lane)?; @@ -284,14 +305,12 @@ impl PatchedArray { let begin = (chunks.start * 1024).saturating_sub(self.offset); let end = (chunks.end * 1024) .saturating_sub(self.offset) - .min(self.len); + .min(self.len()); let offset = if chunks.start == 0 { self.offset } else { 0 }; let inner = self.base_array().slice(begin..end)?; - let len = end - begin; - Ok(PatchedArray { slots: vec![ Some(inner), @@ -301,8 +320,6 @@ impl PatchedArray { ], n_lanes: self.n_lanes, offset, - len, - stats_set: ArrayStats::default(), }) } } diff --git a/vortex-array/src/arrays/patched/compute/compare.rs b/vortex-array/src/arrays/patched/compute/compare.rs index 0ec4448da37..54c89c1744d 100644 --- a/vortex-array/src/arrays/patched/compute/compare.rs +++ b/vortex-array/src/arrays/patched/compute/compare.rs @@ -14,7 +14,7 @@ use crate::arrays::BoolArray; use crate::arrays::ConstantArray; use crate::arrays::Patched; use crate::arrays::PrimitiveArray; -use crate::arrays::bool::BoolArrayParts; +use crate::arrays::bool::BoolDataParts; use crate::arrays::primitive::NativeValue; use crate::builtins::ArrayBuiltins; use crate::dtype::NativePType; @@ -50,12 +50,13 @@ impl CompareKernel for Patched { .execute::(ctx)? .into_bool(); - let BoolArrayParts { + let result = result.into_parts(); + let BoolDataParts { bits, offset, len, validity, - } = result.into_parts(); + } = result.data.into_parts(); let mut bits = BitBufferMut::from_buffer(bits.unwrap_host().into_mut(), offset, len); @@ -164,7 +165,6 @@ mod tests { use crate::ExecutionCtx; use crate::IntoArray; use crate::LEGACY_SESSION; - use crate::array::Array; use crate::arrays::BoolArray; use crate::arrays::ConstantArray; use crate::arrays::Patched; @@ -191,10 +191,11 @@ mod tests { let mut ctx = ExecutionCtx::new(LEGACY_SESSION.clone()); - let lhs = Array::::try_from_data( - PatchedArray::from_array_and_patches(lhs, &patches, &mut ctx).unwrap(), - ) - .unwrap(); + let lhs = PatchedArray::from_array_and_patches(lhs, &patches, &mut ctx) + .unwrap() + .into_array() + .try_into::() + .unwrap(); let rhs = ConstantArray::new(u32::MAX, 512).into_array(); @@ -259,9 +260,10 @@ mod tests { )?; let mut ctx = ExecutionCtx::new(LEGACY_SESSION.clone()); - let lhs = Array::::try_from_data(PatchedArray::from_array_and_patches( - lhs, &patches, &mut ctx, - )?)?; + let lhs = PatchedArray::from_array_and_patches(lhs, &patches, &mut ctx)? + .into_array() + .try_into::() + .unwrap(); let rhs = ConstantArray::new(subnormal, 512).into_array(); @@ -292,9 +294,10 @@ mod tests { )?; let mut ctx = ExecutionCtx::new(LEGACY_SESSION.clone()); - let lhs = Array::::try_from_data(PatchedArray::from_array_and_patches( - lhs, &patches, &mut ctx, - )?)?; + let lhs = PatchedArray::from_array_and_patches(lhs, &patches, &mut ctx)? + .into_array() + .try_into::() + .unwrap(); let rhs = ConstantArray::new(0.0f32, 10).into_array(); diff --git a/vortex-array/src/arrays/patched/compute/filter.rs b/vortex-array/src/arrays/patched/compute/filter.rs index 71aee0238ee..0a2060964aa 100644 --- a/vortex-array/src/arrays/patched/compute/filter.rs +++ b/vortex-array/src/arrays/patched/compute/filter.rs @@ -37,7 +37,7 @@ impl FilterReduce for Patched { } }; - let n_chunks = (array.offset + array.len).div_ceil(1024); + let n_chunks = (array.offset + array.len()).div_ceil(1024); // If all chunks already covered, there is nothing to do. if chunk_start == 0 && chunk_stop == n_chunks { diff --git a/vortex-array/src/arrays/patched/compute/take.rs b/vortex-array/src/arrays/patched/compute/take.rs index 6fac5826537..1257eec5169 100644 --- a/vortex-array/src/arrays/patched/compute/take.rs +++ b/vortex-array/src/arrays/patched/compute/take.rs @@ -12,7 +12,7 @@ use crate::array::ArrayView; use crate::arrays::Patched; use crate::arrays::PrimitiveArray; use crate::arrays::dict::TakeExecute; -use crate::arrays::primitive::PrimitiveArrayParts; +use crate::arrays::primitive::PrimitiveDataParts; use crate::dtype::IntegerPType; use crate::dtype::NativePType; use crate::match_each_native_ptype; @@ -35,7 +35,7 @@ impl TakeExecute for Patched { .take(indices.clone())? .execute::(ctx)?; - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer, validity, ptype, @@ -63,7 +63,7 @@ impl TakeExecute for Patched { output.as_mut(), indices.as_slice::(), array.offset, - array.len, + array.len(), array.n_lanes, lane_offsets.as_slice::(), patch_indices.as_slice::(), diff --git a/vortex-array/src/arrays/patched/vtable/mod.rs b/vortex-array/src/arrays/patched/vtable/mod.rs index eb5c887772a..0e592452e27 100644 --- a/vortex-array/src/arrays/patched/vtable/mod.rs +++ b/vortex-array/src/arrays/patched/vtable/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use prost::Message; mod kernels; mod operations; mod slice; @@ -19,13 +20,10 @@ use crate::ArrayEq; use crate::ArrayHash; use crate::ArrayRef; use crate::Canonical; -use crate::DeserializeMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::IntoArray; use crate::Precision; -use crate::ProstMetadata; -use crate::SerializeMetadata; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; @@ -38,7 +36,7 @@ use crate::arrays::patched::array::NUM_SLOTS; use crate::arrays::patched::array::SLOT_NAMES; use crate::arrays::patched::compute::rules::PARENT_RULES; use crate::arrays::patched::vtable::kernels::PARENT_KERNELS; -use crate::arrays::primitive::PrimitiveArrayParts; +use crate::arrays::primitive::PrimitiveDataParts; use crate::buffer::BufferHandle; use crate::builders::ArrayBuilder; use crate::builders::PrimitiveBuilder; @@ -47,7 +45,6 @@ use crate::dtype::NativePType; use crate::dtype::PType; use crate::match_each_native_ptype; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::vtable; vtable!(Patched); @@ -80,30 +77,13 @@ pub struct PatchedMetadata { impl VTable for Patched { type ArrayData = PatchedArray; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Patched - } - fn id(&self) -> ArrayId { ArrayId::new_ref("vortex.patched") } - fn len(array: &Self::ArrayData) -> usize { - array.len - } - - fn dtype(array: &Self::ArrayData) -> &DType { - array.base_array().dtype() - } - - fn stats(array: &Self::ArrayData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &PatchedArray, state: &mut H, precision: Precision) { array.offset.hash(state); array.n_lanes.hash(state); @@ -132,6 +112,28 @@ impl VTable for Patched { 0 } + fn validate(&self, data: &PatchedArray, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.base_array().dtype() == dtype, + "PatchedArray base dtype {} does not match outer dtype {}", + data.base_array().dtype(), + dtype + ); + vortex_ensure!( + data.len() == len, + "PatchedArray base len {} does not match outer len {}", + data.len(), + len + ); + vortex_ensure!( + data.patch_indices().len() == data.patch_values().len(), + "PatchedArray patch indices len {} does not match patch values len {}", + data.patch_indices().len(), + data.patch_values().len() + ); + Ok(()) + } + fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { vortex_panic!("invalid buffer index for PatchedArray: {idx}"); } @@ -150,27 +152,45 @@ impl VTable for Patched { } } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(PatchedMetadata { - n_patches: u32::try_from(array.patch_indices().len())?, - n_lanes: u32::try_from(array.n_lanes)?, - offset: u32::try_from(array.offset)?, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some( + PatchedMetadata { + n_patches: u32::try_from(array.patch_indices().len())?, + n_lanes: u32::try_from(array.n_lanes)?, + offset: u32::try_from(array.offset)?, + } + .encode_to_vec(), + )) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, + &self, + dtype: &DType, + len: usize, + metadata: &[u8], _buffers: &[BufferHandle], + children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { - let inner = as DeserializeMetadata>::deserialize(bytes)?; - Ok(ProstMetadata(inner)) + ) -> VortexResult { + let metadata = PatchedMetadata::decode(metadata)?; + let n_patches = metadata.n_patches as usize; + let n_lanes = metadata.n_lanes as usize; + let offset = metadata.offset as usize; + + // n_chunks should correspond to the chunk in the `inner`. + // After slicing when offset > 0, there may be additional chunks. + let n_chunks = (len + offset).div_ceil(1024); + + let inner = children.get(0, dtype, len)?; + let lane_offsets = children.get(1, PType::U32.into(), n_chunks * n_lanes + 1)?; + let indices = children.get(2, PType::U16.into(), n_patches)?; + let values = children.get(3, dtype, n_patches)?; + + Ok(PatchedArray { + slots: vec![Some(inner), Some(lane_offsets), Some(indices), Some(values)], + n_lanes, + offset, + }) } fn append_to_builder( @@ -236,35 +256,6 @@ impl VTable for Patched { Ok(()) } - fn build( - dtype: &DType, - len: usize, - metadata: &Self::Metadata, - _buffers: &[BufferHandle], - children: &dyn ArrayChildren, - ) -> VortexResult { - let n_patches = metadata.n_patches as usize; - let n_lanes = metadata.n_lanes as usize; - let offset = metadata.offset as usize; - - // n_chunks should correspond to the chunk in the `inner`. - // After slicing when offset > 0, there may be additional chunks. - let n_chunks = (len + offset).div_ceil(1024); - - let inner = children.get(0, dtype, len)?; - let lane_offsets = children.get(1, PType::U32.into(), n_chunks * n_lanes + 1)?; - let indices = children.get(2, PType::U16.into(), n_patches)?; - let values = children.get(3, dtype, n_patches)?; - - Ok(PatchedArray { - slots: vec![Some(inner), Some(lane_offsets), Some(indices), Some(values)], - n_lanes, - offset, - len, - stats_set: ArrayStats::default(), - }) - } - fn slots(array: ArrayView<'_, Self>) -> &[Option] { &array.data().slots } @@ -291,7 +282,7 @@ impl VTable for Patched { .execute::(ctx)? .into_primitive(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer, ptype, validity, @@ -314,7 +305,7 @@ impl VTable for Patched { let patched_values = match_each_native_ptype!(values.ptype(), |V| { let offset = array.offset; - let len = array.len; + let len = array.len(); let mut output = Buffer::::from_byte_buffer(buffer.unwrap_host()).into_mut(); @@ -404,8 +395,8 @@ mod tests { use crate::assert_arrays_eq; use crate::builders::builder_with_capacity; use crate::patches::Patches; - use crate::serde::ArrayParts; use crate::serde::SerializeOptions; + use crate::serde::SerializedArray; use crate::validity::Validity; #[test] @@ -631,7 +622,7 @@ mod tests { } let concat = concat.freeze(); - let parts = ArrayParts::try_from(concat).unwrap(); + let parts = SerializedArray::try_from(concat).unwrap(); let decoded = parts .decode( &dtype, diff --git a/vortex-array/src/arrays/patched/vtable/slice.rs b/vortex-array/src/arrays/patched/vtable/slice.rs index 71d68585ebe..bbdb4df2464 100644 --- a/vortex-array/src/arrays/patched/vtable/slice.rs +++ b/vortex-array/src/arrays/patched/vtable/slice.rs @@ -11,15 +11,12 @@ use crate::array::ArrayView; use crate::arrays::Patched; use crate::arrays::PatchedArray; use crate::arrays::slice::SliceReduce; -use crate::stats::ArrayStats; impl SliceReduce for Patched { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { // We **always** slice the patches at 1024-element chunk boundaries. We keep the offset + len // around so that when we execute we know how much to chop off. let new_offset = (range.start + array.offset) % 1024; - let new_len = range.end - range.start; - let chunk_start = (range.start + array.offset) / 1024; let chunk_stop = (range.end + array.offset).div_ceil(1024); let sliced_lane_offsets = array @@ -41,8 +38,6 @@ impl SliceReduce for Patched { ], n_lanes: array.n_lanes, offset: new_offset, - len: new_len, - stats_set: ArrayStats::default(), } .into_array(), )) diff --git a/vortex-array/src/arrays/primitive/array/cast.rs b/vortex-array/src/arrays/primitive/array/cast.rs index c92977616ff..f213f6a60c5 100644 --- a/vortex-array/src/arrays/primitive/array/cast.rs +++ b/vortex-array/src/arrays/primitive/array/cast.rs @@ -61,6 +61,20 @@ impl PrimitiveData { } impl PrimitiveArray { + pub fn reinterpret_cast(&self, ptype: PType) -> Self { + if self.ptype() == ptype { + return self.clone(); + } + + assert_eq!( + self.ptype().byte_width(), + ptype.byte_width(), + "can't reinterpret cast between integers of two different widths" + ); + + PrimitiveArray::from_buffer_handle(self.buffer_handle().clone(), ptype, self.validity()) + } + /// Narrow the array to the smallest possible integer type that can represent all values. pub fn narrow(&self) -> VortexResult { if !self.ptype().is_int() { diff --git a/vortex-array/src/arrays/primitive/array/mod.rs b/vortex-array/src/arrays/primitive/array/mod.rs index 1a7a9a0efdd..25cbf8c755b 100644 --- a/vortex-array/src/arrays/primitive/array/mod.rs +++ b/vortex-array/src/arrays/primitive/array/mod.rs @@ -15,13 +15,13 @@ use vortex_error::vortex_panic; use crate::ToCanonical; use crate::array::Array; +use crate::array::ArrayParts; use crate::arrays::Primitive; use crate::dtype::DType; use crate::dtype::NativePType; use crate::dtype::Nullability; use crate::dtype::PType; use crate::match_each_native_ptype; -use crate::stats::ArrayStats; use crate::validity::Validity; mod accessor; @@ -76,12 +76,12 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; #[derive(Clone, Debug)] pub struct PrimitiveData { pub(super) slots: Vec>, - pub(super) dtype: DType, + pub(super) ptype: PType, + pub(super) nullability: Nullability, pub(super) buffer: BufferHandle, - pub(super) stats_set: ArrayStats, } -pub struct PrimitiveArrayParts { +pub struct PrimitiveDataParts { pub ptype: PType, pub buffer: BufferHandle, pub validity: Validity, @@ -107,12 +107,11 @@ impl PrimitiveData { ) -> Self { let len = handle.len() / ptype.byte_width(); let slots = Self::make_slots(&validity, len); - let dtype = DType::Primitive(ptype, validity.nullability()); Self { slots, buffer: handle, - dtype, - stats_set: ArrayStats::default(), + ptype, + nullability: validity.nullability(), } } @@ -163,12 +162,11 @@ impl PrimitiveData { let len = buffer.len(); let slots = Self::make_slots(&validity, len); - let dtype = DType::Primitive(T::PTYPE, validity.nullability()); Self { slots, - dtype, + ptype: T::PTYPE, + nullability: validity.nullability(), buffer: BufferHandle::new_host(buffer.into_byte_buffer()), - stats_set: Default::default(), } } @@ -197,8 +195,10 @@ impl PrimitiveData { impl Array { pub fn empty(nullability: Nullability) -> Self { - Array::try_from_data(PrimitiveData::empty::(nullability)) - .vortex_expect("PrimitiveData is always valid") + let dtype = DType::Primitive(T::PTYPE, nullability); + let len = 0; + let data = PrimitiveData::empty::(nullability); + unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } } /// Creates a new `PrimitiveArray`. @@ -207,13 +207,19 @@ impl Array { /// /// Panics if the provided components do not satisfy the invariants. pub fn new(buffer: impl Into>, validity: Validity) -> Self { - Array::try_from_data(PrimitiveData::new(buffer, validity)) - .vortex_expect("PrimitiveData is always valid") + let buffer = buffer.into(); + let dtype = DType::Primitive(T::PTYPE, validity.nullability()); + let len = buffer.len(); + let data = PrimitiveData::new(buffer, validity); + unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } } /// Constructs a new `PrimitiveArray`. pub fn try_new(buffer: Buffer, validity: Validity) -> VortexResult { - Array::try_from_data(PrimitiveData::try_new(buffer, validity)?) + let dtype = DType::Primitive(T::PTYPE, validity.nullability()); + let len = buffer.len(); + let data = PrimitiveData::try_new(buffer, validity)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) }) } /// Creates a new `PrimitiveArray` without validation. @@ -222,8 +228,10 @@ impl Array { /// /// See [`PrimitiveData::new_unchecked`]. pub unsafe fn new_unchecked(buffer: Buffer, validity: Validity) -> Self { - Array::try_from_data(unsafe { PrimitiveData::new_unchecked(buffer, validity) }) - .vortex_expect("PrimitiveData is always valid") + let dtype = DType::Primitive(T::PTYPE, validity.nullability()); + let len = buffer.len(); + let data = unsafe { PrimitiveData::new_unchecked(buffer, validity) }; + unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } } /// Create a new array from a buffer handle. @@ -236,22 +244,27 @@ impl Array { ptype: PType, validity: Validity, ) -> Self { - Array::try_from_data(unsafe { - PrimitiveData::new_unchecked_from_handle(handle, ptype, validity) - }) - .vortex_expect("PrimitiveData is always valid") + let dtype = DType::Primitive(ptype, validity.nullability()); + let len = handle.len() / ptype.byte_width(); + let data = unsafe { PrimitiveData::new_unchecked_from_handle(handle, ptype, validity) }; + unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } } /// Creates a new `PrimitiveArray` from a [`BufferHandle`]. pub fn from_buffer_handle(handle: BufferHandle, ptype: PType, validity: Validity) -> Self { - Array::try_from_data(PrimitiveData::from_buffer_handle(handle, ptype, validity)) + let dtype = DType::Primitive(ptype, validity.nullability()); + let len = handle.len() / ptype.byte_width(); + let data = PrimitiveData::from_buffer_handle(handle, ptype, validity); + Array::try_from_parts(ArrayParts::new(Primitive, dtype, len, data)) .vortex_expect("PrimitiveData is always valid") } /// Creates a new `PrimitiveArray` from a [`ByteBuffer`]. pub fn from_byte_buffer(buffer: ByteBuffer, ptype: PType, validity: Validity) -> Self { - Array::try_from_data(PrimitiveData::from_byte_buffer(buffer, ptype, validity)) - .vortex_expect("PrimitiveData is always valid") + let dtype = DType::Primitive(ptype, validity.nullability()); + let len = buffer.len() / ptype.byte_width(); + let data = PrimitiveData::from_byte_buffer(buffer, ptype, validity); + unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } } /// Create a PrimitiveArray from a byte buffer containing only the valid elements. @@ -261,13 +274,11 @@ impl Array { validity: Validity, n_rows: usize, ) -> Self { - Array::try_from_data(PrimitiveData::from_values_byte_buffer( - valid_elems_buffer, - ptype, - validity, - n_rows, - )) - .vortex_expect("PrimitiveData is always valid") + let dtype = DType::Primitive(ptype, validity.nullability()); + let len = n_rows; + let data = + PrimitiveData::from_values_byte_buffer(valid_elems_buffer, ptype, validity, n_rows); + unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } } /// Validates the components that would be used to create a `PrimitiveArray`. @@ -278,10 +289,10 @@ impl Array { impl PrimitiveData { /// Consume the primitive array and returns its component parts. - pub fn into_parts(self) -> PrimitiveArrayParts { + pub fn into_parts(self) -> PrimitiveDataParts { let ptype = self.ptype(); let validity = self.validity(); - PrimitiveArrayParts { + PrimitiveDataParts { ptype, buffer: self.buffer, validity, @@ -291,8 +302,8 @@ impl PrimitiveData { impl PrimitiveData { /// Returns the dtype of the array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + DType::Primitive(self.ptype, self.nullability) } /// Returns the length of the array. @@ -308,7 +319,7 @@ impl PrimitiveData { /// Reconstructs the validity from the slot state. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) } /// Returns the validity as a [`Mask`](vortex_mask::Mask). @@ -317,7 +328,7 @@ impl PrimitiveData { } pub fn ptype(&self) -> PType { - self.dtype().as_ptype() + self.ptype } /// Get access to the buffer handle backing the array. @@ -326,14 +337,13 @@ impl PrimitiveData { } pub fn from_buffer_handle(handle: BufferHandle, ptype: PType, validity: Validity) -> Self { - let dtype = DType::Primitive(ptype, validity.nullability()); let len = handle.len() / ptype.byte_width(); let slots = Self::make_slots(&validity, len); Self { slots, buffer: handle, - dtype, - stats_set: ArrayStats::default(), + ptype, + nullability: validity.nullability(), } } diff --git a/vortex-array/src/arrays/primitive/compute/between.rs b/vortex-array/src/arrays/primitive/compute/between.rs index 09ea69e6d16..251aa9c9e7c 100644 --- a/vortex-array/src/arrays/primitive/compute/between.rs +++ b/vortex-array/src/arrays/primitive/compute/between.rs @@ -33,7 +33,7 @@ impl BetweenKernel for Primitive { // null values let nullability = - arr.dtype.nullability() | lower.dtype().nullability() | upper.dtype().nullability(); + arr.dtype().nullability() | lower.dtype().nullability() | upper.dtype().nullability(); Ok(Some(match_each_native_ptype!(arr.ptype(), |P| { between_impl::

( diff --git a/vortex-array/src/arrays/primitive/mod.rs b/vortex-array/src/arrays/primitive/mod.rs index 522a03afc86..7c526c2cfbb 100644 --- a/vortex-array/src/arrays/primitive/mod.rs +++ b/vortex-array/src/arrays/primitive/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::PrimitiveArrayParts; pub use array::PrimitiveData; +pub use array::PrimitiveDataParts; pub use array::chunk_range; pub use array::patch_chunk; pub use vtable::PrimitiveArray; diff --git a/vortex-array/src/arrays/primitive/vtable/mod.rs b/vortex-array/src/arrays/primitive/vtable/mod.rs index db8bd8c5258..ab29d6305f5 100644 --- a/vortex-array/src/arrays/primitive/vtable/mod.rs +++ b/vortex-array/src/arrays/primitive/vtable/mod.rs @@ -8,7 +8,6 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use crate::ArrayRef; -use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::array::Array; @@ -37,37 +36,19 @@ use crate::arrays::primitive::array::SLOT_NAMES; use crate::arrays::primitive::compute::rules::RULES; use crate::hash::ArrayEq; use crate::hash::ArrayHash; -use crate::stats::ArrayStats; vtable!(Primitive, Primitive, PrimitiveData); impl VTable for Primitive { type ArrayData = PrimitiveData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Primitive - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &PrimitiveData) -> usize { - array.buffer_handle().len() / array.ptype().byte_width() - } - - fn dtype(array: &PrimitiveData) -> &DType { - &array.dtype - } - - fn stats(array: &PrimitiveData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &PrimitiveData, state: &mut H, precision: Precision) { array.buffer.array_hash(state, precision); array.validity().array_hash(state, precision); @@ -96,31 +77,45 @@ impl VTable for Primitive { } } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } - fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) + fn validate(&self, data: &PrimitiveData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.len() == len, + "PrimitiveArray length {} does not match outer length {}", + data.len(), + len + ); + + let actual_dtype = data.dtype(); + vortex_ensure!( + &actual_dtype == dtype, + "PrimitiveArray dtype {} does not match outer dtype {}", + actual_dtype, + dtype + ); + + Ok(()) } - fn build( + fn deserialize( + &self, dtype: &DType, len: usize, - _metadata: &Self::Metadata, + metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + if !metadata.is_empty() { + vortex_bail!( + "PrimitiveArray expects empty metadata, got {} bytes", + metadata.len() + ); + } if buffers.len() != 1 { vortex_bail!("Expected 1 buffer, got {}", buffers.len()); } @@ -227,8 +222,8 @@ mod tests { use crate::LEGACY_SESSION; use crate::arrays::PrimitiveArray; use crate::assert_arrays_eq; - use crate::serde::ArrayParts; use crate::serde::SerializeOptions; + use crate::serde::SerializedArray; use crate::validity::Validity; #[test] @@ -251,7 +246,7 @@ mod tests { for buf in serialized { concat.extend_from_slice(buf.as_ref()); } - let parts = ArrayParts::try_from(concat.freeze()).unwrap(); + let parts = SerializedArray::try_from(concat.freeze()).unwrap(); let decoded = parts .decode( &dtype, diff --git a/vortex-array/src/arrays/scalar_fn/array.rs b/vortex-array/src/arrays/scalar_fn/array.rs index 029ff0802fa..1fc9accc886 100644 --- a/vortex-array/src/arrays/scalar_fn/array.rs +++ b/vortex-array/src/arrays/scalar_fn/array.rs @@ -7,20 +7,16 @@ use vortex_error::vortex_ensure; use crate::ArrayRef; use crate::array::Array; +use crate::array::ArrayParts; use crate::arrays::ScalarFnVTable; -use crate::dtype::DType; use crate::scalar_fn::ScalarFnRef; -use crate::stats::ArrayStats; // ScalarFnArray has a variable number of slots (one per child) #[derive(Clone, Debug)] pub struct ScalarFnData { - pub(super) vtable: ScalarFnVTable, - pub(super) dtype: DType, - pub(super) len: usize, + pub(super) scalar_fn: ScalarFnRef, pub(super) slots: Vec>, - pub(super) stats: ArrayStats, } impl ScalarFnData { @@ -30,9 +26,6 @@ impl ScalarFnData { children: Vec, len: usize, ) -> VortexResult { - let arg_dtypes: Vec<_> = children.iter().map(|c| c.dtype().clone()).collect(); - let dtype = scalar_fn.return_dtype(&arg_dtypes)?; - vortex_ensure!( children.iter().all(|c| c.len() == len), "ScalarFnArray must have children equal to the array length" @@ -40,35 +33,14 @@ impl ScalarFnData { let slots = children.into_iter().map(Some).collect(); - Ok(Self { - vtable: ScalarFnVTable { scalar_fn }, - dtype, - len, - slots, - stats: Default::default(), - }) - } - - /// Returns the dtype of the array. - pub fn dtype(&self) -> &DType { - &self.dtype - } - - /// Returns the length of the array. - pub fn len(&self) -> usize { - self.len - } - - /// Returns `true` if the array is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 + Ok(Self { scalar_fn, slots }) } /// Get the scalar function bound to this array. #[allow(clippy::same_name_method)] #[inline(always)] pub fn scalar_fn(&self) -> &ScalarFnRef { - &self.vtable.scalar_fn + &self.scalar_fn } /// Get a child array by index. @@ -116,6 +88,10 @@ impl Array { children: Vec, len: usize, ) -> VortexResult { - Array::try_from_data(ScalarFnData::try_new(scalar_fn, children, len)?) + let arg_dtypes: Vec<_> = children.iter().map(|c| c.dtype().clone()).collect(); + let dtype = scalar_fn.return_dtype(&arg_dtypes)?; + let data = ScalarFnData::try_new(scalar_fn.clone(), children, len)?; + let vtable = ScalarFnVTable { scalar_fn }; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(vtable, dtype, len, data)) }) } } diff --git a/vortex-array/src/arrays/scalar_fn/metadata.rs b/vortex-array/src/arrays/scalar_fn/metadata.rs deleted file mode 100644 index 8a0817bf089..00000000000 --- a/vortex-array/src/arrays/scalar_fn/metadata.rs +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::fmt::Debug; -use std::fmt::Formatter; - -use crate::dtype::DType; -use crate::scalar_fn::ScalarFnRef; - -#[derive(Clone)] -pub struct ScalarFnMetadata { - pub(super) scalar_fn: ScalarFnRef, - pub(super) child_dtypes: Vec, -} - -// Array tree display wrongly uses debug... -impl Debug for ScalarFnMetadata { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.scalar_fn.options()) - } -} diff --git a/vortex-array/src/arrays/scalar_fn/mod.rs b/vortex-array/src/arrays/scalar_fn/mod.rs index d19d1157d0b..778075b9fe0 100644 --- a/vortex-array/src/arrays/scalar_fn/mod.rs +++ b/vortex-array/src/arrays/scalar_fn/mod.rs @@ -2,7 +2,6 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -mod metadata; mod rules; mod vtable; diff --git a/vortex-array/src/arrays/scalar_fn/rules.rs b/vortex-array/src/arrays/scalar_fn/rules.rs index 40fec2594cf..2c1aa67cdaa 100644 --- a/vortex-array/src/arrays/scalar_fn/rules.rs +++ b/vortex-array/src/arrays/scalar_fn/rules.rs @@ -19,7 +19,6 @@ use crate::arrays::ScalarFnArray; use crate::arrays::ScalarFnVTable; use crate::arrays::Slice; use crate::arrays::StructArray; -use crate::arrays::scalar_fn::ScalarFnData; use crate::dtype::DType; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ArrayReduceRule; @@ -61,7 +60,7 @@ impl ArrayReduceRule for ScalarFnPackToStructRule { StructArray::try_new( pack_options.names.clone(), array.children(), - array.len, + array.len(), validity, )? .into_array(), @@ -80,7 +79,7 @@ impl ArrayReduceRule for ScalarFnConstantRule { Ok(Some(Canonical::empty(array.dtype()).into_array())) } else { let result = array.array().scalar_at(0)?; - Ok(Some(ConstantArray::new(result, array.len).into_array())) + Ok(Some(ConstantArray::new(result, array.len()).into_array())) } } } @@ -104,7 +103,7 @@ impl ArrayParentReduceRule for ScalarFnSliceReduceRule { .collect::>()?; Ok(Some( - ScalarFnData::try_new(array.scalar_fn().clone(), children, range.len())?.into_array(), + ScalarFnArray::try_new(array.scalar_fn().clone(), children, range.len())?.into_array(), )) } } @@ -115,7 +114,7 @@ impl ArrayReduceRule for ScalarFnAbstractReduceRule { fn reduce(&self, array: ArrayView<'_, ScalarFnVTable>) -> VortexResult> { if let Some(reduced) = array .scalar_fn() - .reduce(array.as_ref(), &ArrayReduceCtx { len: array.len })? + .reduce(array.as_ref(), &ArrayReduceCtx { len: array.len() })? { return Ok(Some( reduced diff --git a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs index 5f4d3e2835a..c74a53d7730 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs @@ -23,10 +23,10 @@ use crate::IntoArray; use crate::Precision; use crate::array::Array; use crate::array::ArrayId; +use crate::array::ArrayParts; use crate::array::ArrayView; use crate::array::VTable; use crate::arrays::scalar_fn::array::ScalarFnData; -use crate::arrays::scalar_fn::metadata::ScalarFnMetadata; use crate::arrays::scalar_fn::rules::PARENT_RULES; use crate::arrays::scalar_fn::rules::RULES; use crate::buffer::BufferHandle; @@ -44,7 +44,6 @@ use crate::scalar_fn::ScalarFnRef; use crate::scalar_fn::ScalarFnVTableExt; use crate::scalar_fn::VecExecutionArgs; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::vtable; vtable!(ScalarFn, ScalarFnVTable, ScalarFnData); @@ -56,28 +55,32 @@ pub struct ScalarFnVTable { impl VTable for ScalarFnVTable { type ArrayData = ScalarFnData; - type Metadata = ScalarFnMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(array: &ScalarFnData) -> &Self { - &array.vtable - } - fn id(&self) -> ArrayId { self.scalar_fn.id() } - fn len(array: &ScalarFnData) -> usize { - array.len - } - - fn dtype(array: &ScalarFnData) -> &DType { - &array.dtype - } + fn validate(&self, data: &ScalarFnData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.scalar_fn == self.scalar_fn, + "ScalarFnArray data scalar_fn does not match vtable" + ); + vortex_ensure!( + data.iter_children().all(|c| c.len() == len), + "All child arrays must have the same length as the scalar function array" + ); - fn stats(array: &ScalarFnData) -> &ArrayStats { - &array.stats + let child_dtypes = data + .iter_children() + .map(|c| c.dtype().clone()) + .collect_vec(); + vortex_ensure!( + self.scalar_fn.return_dtype(&child_dtypes)? == *dtype, + "ScalarFnArray dtype does not match scalar function return dtype" + ); + Ok(()) } fn array_hash(array: &ScalarFnData, state: &mut H, precision: Precision) { @@ -111,61 +114,22 @@ impl VTable for ScalarFnVTable { None } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - let child_dtypes = array.iter_children().map(|c| c.dtype().clone()).collect(); - Ok(ScalarFnMetadata { - scalar_fn: array.scalar_fn().clone(), - child_dtypes, - }) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { // Not supported Ok(None) } fn deserialize( - _bytes: &[u8], + &self, _dtype: &DType, _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - vortex_bail!("Deserialization of ScalarFnVTable metadata is not supported"); - } + _metadata: &[u8], - fn build( - dtype: &DType, - len: usize, - metadata: &ScalarFnMetadata, _buffers: &[BufferHandle], - children: &dyn ArrayChildren, + _children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { - let children: Vec<_> = metadata - .child_dtypes - .iter() - .enumerate() - .map(|(idx, child_dtype)| children.get(idx, child_dtype, len)) - .try_collect()?; - - #[cfg(debug_assertions)] - { - let child_dtypes: Vec<_> = children.iter().map(|c| c.dtype().clone()).collect(); - vortex_error::vortex_ensure!( - &metadata.scalar_fn.return_dtype(&child_dtypes)? == dtype, - "Return dtype mismatch when building ScalarFnArray" - ); - } - - Ok(ScalarFnData { - vtable: ScalarFnVTable { - scalar_fn: metadata.scalar_fn.clone(), - }, - dtype: dtype.clone(), - len, - slots: children.into_iter().map(Some).collect(), - stats: Default::default(), - }) + vortex_bail!("Deserialization of ScalarFnVTable metadata is not supported"); } fn slots(array: ArrayView<'_, Self>) -> &[Option] { @@ -227,14 +191,15 @@ pub trait ScalarFnArrayExt: scalar_fn::ScalarFnVTable { let child_dtypes = children.iter().map(|c| c.dtype().clone()).collect_vec(); let dtype = scalar_fn.return_dtype(&child_dtypes)?; - Ok(ScalarFnData { - vtable: ScalarFnVTable { scalar_fn }, - dtype, - len, + let data = ScalarFnData { + scalar_fn: scalar_fn.clone(), slots: children.into_iter().map(Some).collect(), - stats: Default::default(), - } - .into_array()) + }; + let vtable = ScalarFnVTable { scalar_fn }; + Ok( + unsafe { Array::from_parts_unchecked(ArrayParts::new(vtable, dtype, len, data)) } + .into_array(), + ) } } impl ScalarFnArrayExt for V {} diff --git a/vortex-array/src/arrays/scalar_fn/vtable/operations.rs b/vortex-array/src/arrays/scalar_fn/vtable/operations.rs index f02f0b7f9bf..c697d784ea8 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/operations.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/operations.rs @@ -44,11 +44,11 @@ impl OperationsVTable for ScalarFnVTable { debug_assert_eq!( scalar.dtype(), - &array.dtype, + array.dtype(), "Scalar function {} returned dtype {:?} but expected {:?}", array.scalar_fn(), scalar.dtype(), - array.dtype + array.dtype() ); Ok(scalar) diff --git a/vortex-array/src/arrays/shared/array.rs b/vortex-array/src/arrays/shared/array.rs index c0d4ab33f86..cf4df660520 100644 --- a/vortex-array/src/arrays/shared/array.rs +++ b/vortex-array/src/arrays/shared/array.rs @@ -14,9 +14,9 @@ use crate::ArrayRef; use crate::Canonical; use crate::IntoArray; use crate::array::Array; +use crate::array::ArrayParts; use crate::arrays::Shared; use crate::dtype::DType; -use crate::stats::ArrayStats; /// The source array that is shared and lazily computed. pub(super) const SOURCE_SLOT: usize = 0; @@ -32,18 +32,14 @@ pub struct SharedData { pub(super) slots: Vec>, cached: Arc>>, async_compute_lock: Arc>, - pub(super) dtype: DType, - pub(super) stats: ArrayStats, } impl SharedData { pub fn new(source: ArrayRef) -> Self { Self { - dtype: source.dtype().clone(), slots: vec![Some(source)], cached: Arc::new(OnceLock::new()), async_compute_lock: Arc::new(AsyncMutex::new(())), - stats: ArrayStats::default(), } } @@ -113,7 +109,7 @@ impl SharedData { /// Returns the [`DType`] of this array. pub fn dtype(&self) -> &DType { - &self.dtype + self.current_array_ref().dtype() } /// Returns `true` if this array is empty. @@ -125,15 +121,21 @@ impl SharedData { impl Array { /// Creates a new `SharedArray`. pub fn new(source: ArrayRef) -> Self { - Array::try_from_data(SharedData::new(source)).vortex_expect("SharedData is always valid") + let dtype = source.dtype().clone(); + let len = source.len(); + unsafe { + Array::from_parts_unchecked(ArrayParts::new( + Shared, + dtype, + len, + SharedData::new(source), + )) + } } } impl SharedData { pub(super) fn set_source(&mut self, source: Option) { - if let Some(ref s) = source { - self.dtype = s.dtype().clone(); - } self.slots = vec![source]; self.cached = Arc::new(OnceLock::new()); self.async_compute_lock = Arc::new(AsyncMutex::new(())); diff --git a/vortex-array/src/arrays/shared/vtable.rs b/vortex-array/src/arrays/shared/vtable.rs index 8f04b2a37f9..abd4723ca5d 100644 --- a/vortex-array/src/arrays/shared/vtable.rs +++ b/vortex-array/src/arrays/shared/vtable.rs @@ -8,7 +8,6 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::Canonical; -use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; @@ -25,7 +24,6 @@ use crate::dtype::DType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::scalar::Scalar; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; @@ -42,27 +40,17 @@ impl Shared { impl VTable for Shared { type ArrayData = SharedData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &SharedData) -> &Self { - &Shared - } fn id(&self) -> ArrayId { Self::ID } - fn len(array: &SharedData) -> usize { - array.current_array_ref().len() - } - - fn dtype(array: &SharedData) -> &DType { - &array.dtype - } - - fn stats(array: &SharedData) -> &ArrayStats { - &array.stats + fn validate(&self, data: &SharedData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_error::vortex_ensure!(data.source().dtype() == dtype, "SharedArray dtype mismatch"); + vortex_error::vortex_ensure!(data.source().len() == len, "SharedArray len mismatch"); + Ok(()) } fn array_hash(array: &SharedData, state: &mut H, precision: Precision) { @@ -110,33 +98,21 @@ impl VTable for Shared { Ok(()) } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { vortex_error::vortex_bail!("Shared array is not serializable") } fn deserialize( - _bytes: &[u8], + &self, _dtype: &DType, _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - vortex_error::vortex_bail!("Shared array is not serializable") - } + _metadata: &[u8], - fn build( - dtype: &DType, - len: usize, - _metadata: &Self::Metadata, _buffers: &[BufferHandle], - children: &dyn crate::serde::ArrayChildren, + _children: &dyn crate::serde::ArrayChildren, + _session: &VortexSession, ) -> VortexResult { - let child = children.get(0, dtype, len)?; - Ok(SharedData::new(child)) + vortex_error::vortex_bail!("Shared array is not serializable") } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { diff --git a/vortex-array/src/arrays/slice/array.rs b/vortex-array/src/arrays/slice/array.rs index a8ded19335b..2d438c3fe9c 100644 --- a/vortex-array/src/arrays/slice/array.rs +++ b/vortex-array/src/arrays/slice/array.rs @@ -9,9 +9,9 @@ use vortex_error::vortex_panic; use crate::ArrayRef; use crate::array::Array; +use crate::array::ArrayParts; use crate::arrays::Slice; use crate::dtype::DType; -use crate::stats::ArrayStats; /// The underlying child array being sliced. pub(super) const CHILD_SLOT: usize = 0; @@ -22,16 +22,15 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child"]; pub struct SliceData { pub(super) slots: Vec>, pub(super) range: Range, - pub(super) stats: ArrayStats, } -pub struct SliceArrayParts { +pub struct SliceDataParts { pub child: ArrayRef, pub range: Range, } impl SliceData { - pub fn try_new(child: ArrayRef, range: Range) -> VortexResult { + fn try_new(child: ArrayRef, range: Range) -> VortexResult { if range.end > child.len() { vortex_panic!( "SliceArray range out of bounds: range {:?} exceeds child array length {}", @@ -42,7 +41,6 @@ impl SliceData { Ok(Self { slots: vec![Some(child)], range, - stats: ArrayStats::default(), }) } @@ -76,29 +74,31 @@ impl SliceData { .as_ref() .vortex_expect("SliceArray child slot") } + + pub fn into_parts(mut self) -> SliceDataParts { + SliceDataParts { + child: self.slots[CHILD_SLOT] + .take() + .vortex_expect("SliceArray child slot"), + range: self.range, + } + } } impl Array { /// Constructs a new `SliceArray`. pub fn try_new(child: ArrayRef, range: Range) -> VortexResult { - Array::try_from_data(SliceData::try_new(child, range)?) + let len = range.len(); + let dtype = child.dtype().clone(); + let data = SliceData::try_new(child, range)?; + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Slice, dtype, len, data)) }) } /// Constructs a new `SliceArray`. pub fn new(child: ArrayRef, range: Range) -> Self { - Array::try_from_data(SliceData::new(child, range)) - .vortex_expect("SliceData is always valid") - } -} - -impl SliceData { - /// Consume the slice array and return its components. - pub fn into_parts(mut self) -> SliceArrayParts { - SliceArrayParts { - child: self.slots[CHILD_SLOT] - .take() - .vortex_expect("SliceArray child slot"), - range: self.range, - } + let len = range.len(); + let dtype = child.dtype().clone(); + let data = SliceData::new(child, range); + unsafe { Array::from_parts_unchecked(ArrayParts::new(Slice, dtype, len, data)) } } } diff --git a/vortex-array/src/arrays/slice/mod.rs b/vortex-array/src/arrays/slice/mod.rs index 928367ea213..f701a274a14 100644 --- a/vortex-array/src/arrays/slice/mod.rs +++ b/vortex-array/src/arrays/slice/mod.rs @@ -15,8 +15,8 @@ mod vtable; use std::ops::Range; -pub use array::SliceArrayParts; pub use array::SliceData; +pub use array::SliceDataParts; use vortex_error::VortexResult; pub use vtable::*; diff --git a/vortex-array/src/arrays/slice/vtable.rs b/vortex-array/src/arrays/slice/vtable.rs index c5fa9241001..562fcd4c117 100644 --- a/vortex-array/src/arrays/slice/vtable.rs +++ b/vortex-array/src/arrays/slice/vtable.rs @@ -36,7 +36,6 @@ use crate::executor::ExecutionCtx; use crate::executor::ExecutionResult; use crate::scalar::Scalar; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; @@ -51,27 +50,33 @@ impl Slice { impl VTable for Slice { type ArrayData = SliceData; - type Metadata = SliceMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &SliceData) -> &Self { - &Slice - } fn id(&self) -> ArrayId { Slice::ID } - fn len(array: &SliceData) -> usize { - array.range.len() - } - - fn dtype(array: &SliceData) -> &DType { - array.child().dtype() - } - - fn stats(array: &SliceData) -> &ArrayStats { - &array.stats + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.child().dtype() == dtype, + "SliceArray dtype {} does not match outer dtype {}", + data.child().dtype(), + dtype + ); + vortex_ensure!( + data.len() == len, + "SliceArray length {} does not match outer length {}", + data.len(), + len + ); + vortex_ensure!( + data.range.end <= data.child().len(), + "SliceArray range {:?} exceeds child length {}", + data.range, + data.child().len() + ); + Ok(()) } fn array_hash(array: &SliceData, state: &mut H, precision: Precision) { @@ -104,35 +109,22 @@ impl VTable for Slice { SLOT_NAMES[idx].to_string() } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(SliceMetadata(array.range.clone())) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { // TODO(joe): make this configurable vortex_bail!("Slice array is not serializable") } fn deserialize( - _bytes: &[u8], + &self, _dtype: &DType, _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - vortex_bail!("Slice array is not serializable") - } + _metadata: &[u8], - fn build( - dtype: &DType, - len: usize, - metadata: &SliceMetadata, _buffers: &[BufferHandle], - children: &dyn ArrayChildren, + _children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { - assert_eq!(len, metadata.0.len()); - let child = children.get(0, dtype, metadata.0.end)?; - SliceData::try_new(child, metadata.0.clone()) + vortex_bail!("Slice array is not serializable") } fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { diff --git a/vortex-array/src/arrays/struct_/array.rs b/vortex-array/src/arrays/struct_/array.rs index f8ce8a54bae..3f75a48e958 100644 --- a/vortex-array/src/arrays/struct_/array.rs +++ b/vortex-array/src/arrays/struct_/array.rs @@ -13,6 +13,7 @@ use vortex_error::vortex_err; use crate::ArrayRef; use crate::IntoArray; use crate::array::Array; +use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Struct; @@ -20,7 +21,6 @@ use crate::dtype::DType; use crate::dtype::FieldName; use crate::dtype::FieldNames; use crate::dtype::StructFields; -use crate::stats::ArrayStats; use crate::validity::Validity; // StructArray has a variable number of slots: [validity?, field_0, ..., field_N] @@ -149,13 +149,13 @@ pub(super) const FIELDS_OFFSET: usize = 1; /// ``` #[derive(Clone, Debug)] pub struct StructData { - pub(super) len: usize, - pub(super) dtype: DType, + pub(super) names: FieldNames, + pub(super) nullability: crate::dtype::Nullability, + pub(super) fieldless_len: Option, pub(super) slots: Vec>, - pub(super) stats_set: ArrayStats, } -pub struct StructArrayParts { +pub struct StructDataParts { pub struct_fields: StructFields, pub fields: Arc<[ArrayRef]>, pub validity: Validity, @@ -164,12 +164,15 @@ pub struct StructArrayParts { impl StructData { /// Returns the length of this array. pub fn len(&self) -> usize { - self.len + self.slots[FIELDS_OFFSET..] + .first() + .and_then(|slot| slot.as_ref()) + .map_or_else(|| self.fieldless_len.unwrap_or(0), |field| field.len()) } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + DType::Struct(self.struct_fields(), self.nullability) } /// Returns `true` if this array is empty. @@ -179,7 +182,7 @@ impl StructData { /// Reconstructs the validity from the slots. pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) } /// Return an iterator over the struct fields without the validity of the struct applied. @@ -215,7 +218,7 @@ impl StructData { /// Return the struct field without the validity of the struct applied pub fn unmasked_field_by_name_opt(&self, name: impl AsRef) -> Option<&ArrayRef> { let name = name.as_ref(); - self.struct_fields().find(name).map(|idx| { + self.names.find(name).map(|idx| { self.slots[FIELDS_OFFSET + idx] .as_ref() .vortex_expect("StructArray field slot") @@ -223,16 +226,16 @@ impl StructData { } pub fn names(&self) -> &FieldNames { - self.struct_fields().names() + &self.names } - pub fn struct_fields(&self) -> &StructFields { - let Some(struct_dtype) = &self.dtype.as_struct_fields_opt() else { - unreachable!( - "struct arrays must have be a DType::Struct, this is likely an internal bug." - ) - }; - struct_dtype + pub fn struct_fields(&self) -> StructFields { + StructFields::new( + self.names.clone(), + self.iter_unmasked_fields() + .map(|field| field.dtype().clone()) + .collect(), + ) } /// Create a new `StructArray` with the given length, but without any fields. @@ -270,7 +273,7 @@ impl StructData { /// /// Returns an error if the provided components do not satisfy the invariants documented in /// `StructArray::new_unchecked`. - pub fn try_new( + pub(crate) fn try_new( names: FieldNames, fields: impl Into>, length: usize, @@ -329,10 +332,10 @@ impl StructData { .collect(); Self { - len: length, - dtype: DType::Struct(dtype, validity.nullability()), + names: dtype.names().clone(), + nullability: validity.nullability(), + fieldless_len: fields.is_empty().then_some(length), slots, - stats_set: Default::default(), } } @@ -389,7 +392,7 @@ impl StructData { Ok(()) } - pub fn try_new_with_dtype( + pub(crate) fn try_new_with_dtype( fields: impl Into>, dtype: StructFields, length: usize, @@ -402,16 +405,33 @@ impl StructData { Ok(unsafe { Self::new_unchecked(fields, dtype, length, validity) }) } - pub fn into_parts(self) -> StructArrayParts { - let validity = self.validity(); - let struct_fields = self.dtype.into_struct_fields(); - let fields: Arc<[ArrayRef]> = self - .slots + pub fn into_parts(self) -> StructDataParts { + let StructData { + names, + slots, + nullability, + .. + } = self; + let validity = child_to_validity(&slots[VALIDITY_SLOT], nullability); + let struct_fields = StructFields::new( + names, + slots + .iter() + .skip(FIELDS_OFFSET) + .map(|slot| { + slot.as_ref() + .vortex_expect("StructArray field slot") + .dtype() + .clone() + }) + .collect(), + ); + let fields: Arc<[ArrayRef]> = slots .into_iter() .skip(FIELDS_OFFSET) .map(|s| s.vortex_expect("StructArray field slot")) .collect(); - StructArrayParts { + StructDataParts { struct_fields, fields, validity, @@ -452,47 +472,13 @@ impl StructData { Self::try_from_iter_with_validity(iter, Validity::NonNullable) } - // TODO(aduffy): Add equivalent function to support field masks for nested column access. - /// Return a new StructArray with the given projection applied. - /// - /// Projection does not copy data arrays. Projection is defined by an ordinal array slice - /// which specifies the new ordering of columns in the struct. The projection can be used to - /// perform column re-ordering, deletion, or duplication at a logical level, without any data - /// copying. - pub fn project(&self, projection: &[FieldName]) -> VortexResult { - let mut children = Vec::with_capacity(projection.len()); - let mut names = Vec::with_capacity(projection.len()); - - for f_name in projection.iter() { - let idx = self - .names() - .iter() - .position(|name| name == f_name) - .ok_or_else(|| vortex_err!("Unknown field {f_name}"))?; - - names.push(self.names()[idx].clone()); - children.push( - self.slots[FIELDS_OFFSET + idx] - .as_ref() - .vortex_expect("StructArray field slot") - .clone(), - ); - } - - StructData::try_new( - FieldNames::from(names.as_slice()), - children, - self.len(), - self.validity(), - ) - } - /// Removes and returns a column from the struct array by name. /// If the column does not exist, returns `None`. pub fn remove_column(&mut self, name: impl Into) -> Option { let name = name.into(); - let struct_dtype = self.struct_fields().clone(); + let struct_dtype = self.struct_fields(); + let len = self.len(); let position = struct_dtype .names() @@ -514,7 +500,13 @@ impl StructData { if let Ok(new_dtype) = struct_dtype.without_field(position) { self.slots = new_slots; - self.dtype = DType::Struct(new_dtype, self.dtype.nullability()); + self.names = new_dtype.names().clone(); + self.fieldless_len = self + .slots + .get(FIELDS_OFFSET) + .and_then(|slot| slot.as_ref()) + .map(|_| None) + .unwrap_or(Some(len)); return Some(field); } None @@ -529,8 +521,10 @@ impl Array { length: usize, validity: Validity, ) -> Self { - Array::try_from_data(StructData::new(names, fields, length, validity)) - .vortex_expect("StructData is always valid") + let data = StructData::new(names, fields, length, validity); + let dtype = data.dtype(); + let len = data.len(); + unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) } } /// Constructs a new `StructArray`. @@ -540,7 +534,10 @@ impl Array { length: usize, validity: Validity, ) -> VortexResult { - Array::try_from_data(StructData::try_new(names, fields, length, validity)?) + let data = StructData::try_new(names, fields, length, validity)?; + let dtype = data.dtype(); + let len = data.len(); + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) } /// Creates a new `StructArray` without validation. @@ -554,8 +551,10 @@ impl Array { length: usize, validity: Validity, ) -> Self { - Array::try_from_data(unsafe { StructData::new_unchecked(fields, dtype, length, validity) }) - .vortex_expect("StructData is always valid") + let data = unsafe { StructData::new_unchecked(fields, dtype, length, validity) }; + let dtype = data.dtype(); + let len = data.len(); + unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) } } /// Constructs a new `StructArray` with an explicit dtype. @@ -565,19 +564,18 @@ impl Array { length: usize, validity: Validity, ) -> VortexResult { - Array::try_from_data(StructData::try_new_with_dtype( - fields, dtype, length, validity, - )?) + let data = StructData::try_new_with_dtype(fields, dtype, length, validity)?; + let dtype = data.dtype(); + let len = data.len(); + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) } /// Construct a `StructArray` from named fields. pub fn from_fields>(items: &[(N, ArrayRef)]) -> VortexResult { - Array::try_from_data(StructData::from_fields(items)?) - } - - /// Decompose this struct array into its constituent parts. - pub fn into_parts(self) -> StructArrayParts { - self.into_data().into_parts() + let data = StructData::from_fields(items)?; + let dtype = data.dtype(); + let len = data.len(); + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) } /// Create a `StructArray` from an iterator of (name, array) pairs with validity. @@ -589,27 +587,65 @@ impl Array { iter: T, validity: Validity, ) -> VortexResult { - Array::try_from_data(StructData::try_from_iter_with_validity(iter, validity)?) + let data = StructData::try_from_iter_with_validity(iter, validity)?; + let dtype = data.dtype(); + let len = data.len(); + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) } /// Create a `StructArray` from an iterator of (name, array) pairs. pub fn try_from_iter, A: IntoArray, T: IntoIterator>( iter: T, ) -> VortexResult { - Array::try_from_data(StructData::try_from_iter(iter)?) + let data = StructData::try_from_iter(iter)?; + let dtype = data.dtype(); + let len = data.len(); + Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) + } + + // TODO(aduffy): Add equivalent function to support field masks for nested column access. + /// Return a new StructArray with the given projection applied. + /// + /// Projection does not copy data arrays. Projection is defined by an ordinal array slice + /// which specifies the new ordering of columns in the struct. The projection can be used to + /// perform column re-ordering, deletion, or duplication at a logical level, without any data + /// copying. + pub fn project(&self, projection: &[FieldName]) -> VortexResult { + let mut children = Vec::with_capacity(projection.len()); + let mut names = Vec::with_capacity(projection.len()); + + for f_name in projection { + let idx = self + .names() + .iter() + .position(|name| name == f_name) + .ok_or_else(|| vortex_err!("Unknown field {f_name}"))?; + + names.push(self.names()[idx].clone()); + children.push(self.unmasked_field(idx).clone()); + } + + Self::try_new( + FieldNames::from(names.as_slice()), + children, + self.len(), + self.validity(), + ) } /// Create a fieldless `StructArray` with the given length. pub fn new_fieldless_with_len(len: usize) -> Self { - Array::try_from_data(StructData::new_fieldless_with_len(len)) - .vortex_expect("StructData is always valid") + let data = StructData::new_fieldless_with_len(len); + let dtype = data.dtype(); + let len = data.len(); + unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) } } } impl StructData { pub fn with_column(&self, name: impl Into, array: ArrayRef) -> VortexResult { let name = name.into(); - let struct_dtype = self.struct_fields().clone(); + let struct_dtype = self.struct_fields(); let names = struct_dtype.names().iter().cloned().chain(once(name)); let types = struct_dtype.fields().chain(once(array.dtype().clone())); @@ -621,6 +657,6 @@ impl StructData { .chain(once(array)) .collect(); - Self::try_new_with_dtype(children, new_fields, self.len, self.validity()) + Self::try_new_with_dtype(children, new_fields, self.len(), self.validity()) } } diff --git a/vortex-array/src/arrays/struct_/compute/mask.rs b/vortex-array/src/arrays/struct_/compute/mask.rs index cdf8a772bbf..c17493c11bf 100644 --- a/vortex-array/src/arrays/struct_/compute/mask.rs +++ b/vortex-array/src/arrays/struct_/compute/mask.rs @@ -15,7 +15,7 @@ impl MaskReduce for Struct { fn mask(array: ArrayView<'_, Struct>, mask: &ArrayRef) -> VortexResult> { StructArray::try_new_with_dtype( array.unmasked_fields().iter().cloned().collect::>(), - array.struct_fields().clone(), + array.struct_fields(), array.len(), array.validity().and(Validity::Array(mask.clone()))?, ) diff --git a/vortex-array/src/arrays/struct_/compute/rules.rs b/vortex-array/src/arrays/struct_/compute/rules.rs index ddb471e8930..4d21b190151 100644 --- a/vortex-array/src/arrays/struct_/compute/rules.rs +++ b/vortex-array/src/arrays/struct_/compute/rules.rs @@ -82,7 +82,7 @@ impl ArrayParentReduceRule for StructCastPushDownRule { } else { array .validity() - .into_non_nullable(array.len) + .into_non_nullable(array.len()) .ok_or_else(|| vortex_err!("Failed to cast nullable struct to non-nullable"))? }; diff --git a/vortex-array/src/arrays/struct_/compute/slice.rs b/vortex-array/src/arrays/struct_/compute/slice.rs index 8d4a4a69737..0f36d87c560 100644 --- a/vortex-array/src/arrays/struct_/compute/slice.rs +++ b/vortex-array/src/arrays/struct_/compute/slice.rs @@ -25,7 +25,7 @@ impl SliceReduce for Struct { unsafe { StructArray::new_unchecked( fields, - array.struct_fields().clone(), + array.struct_fields(), range.len(), array.validity().slice(range)?, ) diff --git a/vortex-array/src/arrays/struct_/compute/take.rs b/vortex-array/src/arrays/struct_/compute/take.rs index 8836b45a2aa..8ce8b3f3851 100644 --- a/vortex-array/src/arrays/struct_/compute/take.rs +++ b/vortex-array/src/arrays/struct_/compute/take.rs @@ -20,7 +20,7 @@ impl TakeReduce for Struct { if array.is_empty() { return StructArray::try_new_with_dtype( array.iter_unmasked_fields().cloned().collect::>(), - array.struct_fields().clone(), + array.struct_fields(), indices.len(), Validity::AllInvalid, ) @@ -41,7 +41,7 @@ impl TakeReduce for Struct { .iter_unmasked_fields() .map(|field| field.take(inner_indices.clone())) .collect::, _>>()?, - array.struct_fields().clone(), + array.struct_fields(), indices.len(), array.validity().take(indices)?, ) diff --git a/vortex-array/src/arrays/struct_/compute/zip.rs b/vortex-array/src/arrays/struct_/compute/zip.rs index bf13f4f72cf..75fe4288dd6 100644 --- a/vortex-array/src/arrays/struct_/compute/zip.rs +++ b/vortex-array/src/arrays/struct_/compute/zip.rs @@ -54,7 +54,7 @@ impl ZipKernel for Struct { let combined = (v1m.bitand(&mask_mask)).bitor(&v2m.bitand(&mask_mask.not())); Validity::from_mask( combined, - if_true.dtype.nullability() | if_false.dtype.nullability(), + if_true.dtype().nullability() | if_false.dtype().nullability(), ) } }; diff --git a/vortex-array/src/arrays/struct_/mod.rs b/vortex-array/src/arrays/struct_/mod.rs index b867f903901..65b53e2d603 100644 --- a/vortex-array/src/arrays/struct_/mod.rs +++ b/vortex-array/src/arrays/struct_/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::StructArrayParts; pub use array::StructData; +pub use array::StructDataParts; pub use vtable::StructArray; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/struct_/vtable/mod.rs b/vortex-array/src/arrays/struct_/vtable/mod.rs index 819db769eff..3d32534cd3f 100644 --- a/vortex-array/src/arrays/struct_/vtable/mod.rs +++ b/vortex-array/src/arrays/struct_/vtable/mod.rs @@ -10,7 +10,6 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; -use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::array::Array; @@ -33,36 +32,19 @@ use crate::Precision; use crate::array::ArrayId; use crate::hash::ArrayEq; use crate::hash::ArrayHash; -use crate::stats::ArrayStats; vtable!(Struct, Struct, StructData); impl VTable for Struct { type ArrayData = StructData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Struct - } fn id(&self) -> ArrayId { Self::ID } - fn len(array: &StructData) -> usize { - array.len - } - - fn dtype(array: &StructData) -> &DType { - &array.dtype - } - - fn stats(array: &StructData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &StructData, state: &mut H, precision: Precision) { for field in array.iter_unmasked_fields() { field.array_hash(state, precision); @@ -83,6 +65,29 @@ impl VTable for Struct { 0 } + fn validate(&self, data: &StructData, dtype: &DType, len: usize) -> VortexResult<()> { + match dtype { + DType::Struct(..) => {} + _ => vortex_bail!("Expected struct dtype, found {:?}", dtype), + } + if data.len() != len { + vortex_bail!( + InvalidArgument: "StructArray length {} does not match outer length {}", + data.len(), + len + ); + } + let data_dtype = data.dtype(); + if &data_dtype != dtype { + vortex_bail!( + InvalidArgument: "StructArray dtype {} does not match outer dtype {}", + data_dtype, + dtype + ); + } + Ok(()) + } + fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { vortex_panic!("StructArray buffer index {idx} out of bounds") } @@ -91,31 +96,26 @@ impl VTable for Struct { vortex_panic!("StructArray buffer_name index {idx} out of bounds") } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) - } - - fn build( + &self, dtype: &DType, len: usize, - _metadata: &Self::Metadata, + metadata: &[u8], + _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + if !metadata.is_empty() { + vortex_bail!( + "StructArray expects empty metadata, got {} bytes", + metadata.len() + ); + } let DType::Struct(struct_dtype, nullability) = dtype else { vortex_bail!("Expected struct dtype, found {:?}", dtype) }; diff --git a/vortex-array/src/arrays/varbin/array.rs b/vortex-array/src/arrays/varbin/array.rs index c50c06285ef..329cf3da04c 100644 --- a/vortex-array/src/arrays/varbin/array.rs +++ b/vortex-array/src/arrays/varbin/array.rs @@ -5,6 +5,7 @@ use num_traits::AsPrimitive; use vortex_buffer::ByteBuffer; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_mask::Mask; @@ -12,6 +13,7 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::ToCanonical; use crate::array::Array; +use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::VarBin; @@ -21,7 +23,6 @@ use crate::dtype::DType; use crate::dtype::IntegerPType; use crate::dtype::Nullability; use crate::match_each_integer_ptype; -use crate::stats::ArrayStats; use crate::validity::Validity; /// The offsets array defining the start/end of each variable-length binary element. @@ -33,13 +34,29 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["offsets", "validity"]; #[derive(Clone, Debug)] pub struct VarBinData { - pub(super) dtype: DType, + pub(super) is_utf8: bool, + pub(super) nullability: Nullability, pub(super) bytes: BufferHandle, pub(super) slots: Vec>, - pub(super) stats_set: ArrayStats, } impl VarBinData { + fn dtype_parts(dtype: &DType) -> VortexResult<(bool, Nullability)> { + match dtype { + DType::Utf8(nullability) => Ok((true, *nullability)), + DType::Binary(nullability) => Ok((false, *nullability)), + _ => vortex_bail!(MismatchedTypes: "utf8 or binary", dtype), + } + } + + fn make_dtype(is_utf8: bool, nullability: Nullability) -> DType { + if is_utf8 { + DType::Utf8(nullability) + } else { + DType::Binary(nullability) + } + } + /// Creates a new `VarBinArray`. /// /// # Panics @@ -166,12 +183,14 @@ impl VarBinData { let len = offsets.len().saturating_sub(1); let validity_slot = validity_to_child(&validity, len); + let (is_utf8, nullability) = + Self::dtype_parts(&dtype).vortex_expect("VarBinArray dtype must be utf8 or binary"); Self { - dtype, + is_utf8, + nullability, bytes, slots: vec![Some(offsets), validity_slot], - stats_set: Default::default(), } } @@ -275,8 +294,8 @@ impl VarBinData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + Self::make_dtype(self.is_utf8, self.nullability) } /// Returns `true` if this array is empty. @@ -287,7 +306,7 @@ impl VarBinData { /// Returns the [`Validity`] of this array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) } /// Returns the validity as a [`Mask`]. @@ -381,9 +400,15 @@ impl VarBinData { /// Forwarding constructors for `VarBinArray` (= `Array`). impl Array { + #[inline] + fn from_prevalidated_data(data: VarBinData) -> Self { + let dtype = data.dtype(); + let len = data.len(); + unsafe { Array::from_parts_unchecked(ArrayParts::new(VarBin, dtype, len, data)) } + } + pub fn from_vec>(vec: Vec, dtype: DType) -> Self { - Array::try_from_data(VarBinData::from_vec(vec, dtype)) - .vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from_vec(vec, dtype)) } #[expect( @@ -394,16 +419,14 @@ impl Array { iter: I, dtype: DType, ) -> Self { - Array::try_from_data(VarBinData::from_iter(iter, dtype)) - .vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from_iter(iter, dtype)) } pub fn from_iter_nonnull, I: IntoIterator>( iter: I, dtype: DType, ) -> Self { - Array::try_from_data(VarBinData::from_iter_nonnull(iter, dtype)) - .vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from_iter_nonnull(iter, dtype)) } /// Create from a vector of string slices. @@ -465,15 +488,14 @@ impl VarBinData { let offsets = self.slots[OFFSETS_SLOT] .take() .vortex_expect("VarBinArray offsets slot"); - (self.dtype, self.bytes, offsets, validity) + (self.dtype(), self.bytes, offsets, validity) } } impl Array { /// Creates a new `VarBinArray`. pub fn new(offsets: ArrayRef, bytes: ByteBuffer, dtype: DType, validity: Validity) -> Self { - Array::try_from_data(VarBinData::new(offsets, bytes, dtype, validity)) - .vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::new(offsets, bytes, dtype, validity)) } /// Creates a new `VarBinArray` without validation. @@ -487,8 +509,9 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Array::try_from_data(unsafe { VarBinData::new_unchecked(offsets, bytes, dtype, validity) }) - .vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(unsafe { + VarBinData::new_unchecked(offsets, bytes, dtype, validity) + }) } /// Creates a new `VarBinArray` without validation from a [`BufferHandle`]. @@ -502,10 +525,9 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Array::try_from_data(unsafe { + Self::from_prevalidated_data(unsafe { VarBinData::new_unchecked_from_handle(offsets, bytes, dtype, validity) }) - .vortex_expect("VarBinData is always valid") } /// Constructs a new `VarBinArray`. @@ -515,7 +537,9 @@ impl Array { dtype: DType, validity: Validity, ) -> VortexResult { - Array::try_from_data(VarBinData::try_new(offsets, bytes, dtype, validity)?) + Ok(Self::from_prevalidated_data(VarBinData::try_new( + offsets, bytes, dtype, validity, + )?)) } } @@ -595,76 +619,72 @@ impl<'a> FromIterator> for VarBinData { impl From> for Array { fn from(value: Vec<&[u8]>) -> Self { - Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from(value)) } } impl From>> for Array { fn from(value: Vec>) -> Self { - Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from(value)) } } impl From> for Array { fn from(value: Vec) -> Self { - Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from(value)) } } impl From> for Array { fn from(value: Vec<&str>) -> Self { - Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from(value)) } } impl From>> for Array { fn from(value: Vec>) -> Self { - Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from(value)) } } impl From>>> for Array { fn from(value: Vec>>) -> Self { - Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from(value)) } } impl From>> for Array { fn from(value: Vec>) -> Self { - Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from(value)) } } impl From>> for Array { fn from(value: Vec>) -> Self { - Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") + Self::from_prevalidated_data(VarBinData::from(value)) } } impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Array::try_from_data(>::from_iter(iter)) - .vortex_expect(" is always valid") + Self::from_prevalidated_data(>::from_iter(iter)) } } impl FromIterator>> for Array { fn from_iter>>>(iter: T) -> Self { - Array::try_from_data(>::from_iter(iter)) - .vortex_expect(" is always valid") + Self::from_prevalidated_data(>::from_iter(iter)) } } impl FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Array::try_from_data(>::from_iter(iter)) - .vortex_expect(" is always valid") + Self::from_prevalidated_data(>::from_iter(iter)) } } impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Array::try_from_data(>::from_iter(iter)) - .vortex_expect(" is always valid") + Self::from_prevalidated_data(>::from_iter(iter)) } } diff --git a/vortex-array/src/arrays/varbin/builder.rs b/vortex-array/src/arrays/varbin/builder.rs index 78675e337d7..8dd5066a889 100644 --- a/vortex-array/src/arrays/varbin/builder.rs +++ b/vortex-array/src/arrays/varbin/builder.rs @@ -7,8 +7,6 @@ use vortex_buffer::BufferMut; use vortex_error::vortex_panic; use crate::IntoArray; -use crate::array::VTable; -use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; use crate::arrays::VarBinArray; use crate::dtype::DType; @@ -105,7 +103,9 @@ impl VarBinBuilder { offsets.statistics().compute_is_sorted().unwrap_or(false), "VarBinBuilder offsets must be sorted" ); - Primitive::stats(&offsets).set(Stat::IsSorted, Precision::Exact(true.into())); + offsets + .statistics() + .set(Stat::IsSorted, Precision::Exact(true.into())); // SAFETY: The builder maintains all invariants: // - Offsets are monotonically increasing starting from 0 (guaranteed by builder logic). diff --git a/vortex-array/src/arrays/varbin/vtable/mod.rs b/vortex-array/src/arrays/varbin/vtable/mod.rs index 8b8664ce506..0b6813ddc99 100644 --- a/vortex-array/src/arrays/varbin/vtable/mod.rs +++ b/vortex-array/src/arrays/varbin/vtable/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use prost::Message; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -8,12 +9,9 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use crate::ArrayRef; -use crate::DeserializeMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::IntoArray; -use crate::ProstMetadata; -use crate::SerializeMetadata; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; @@ -41,7 +39,6 @@ use crate::Precision; use crate::arrays::varbin::compute::rules::PARENT_RULES; use crate::hash::ArrayEq; use crate::hash::ArrayHash; -use crate::stats::ArrayStats; vtable!(VarBin, VarBin, VarBinData); @@ -54,29 +51,13 @@ pub struct VarBinMetadata { impl VTable for VarBin { type ArrayData = VarBinData; - type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &VarBinData) -> &Self { - &VarBin - } fn id(&self) -> ArrayId { Self::ID } - fn len(array: &VarBinData) -> usize { - array.offsets().len().saturating_sub(1) - } - - fn dtype(array: &VarBinData) -> &DType { - &array.dtype - } - - fn stats(array: &VarBinData) -> &ArrayStats { - &array.stats_set - } - fn array_hash(array: &VarBinData, state: &mut H, precision: Precision) { array.bytes().array_hash(state, precision); array.offsets().array_hash(state, precision); @@ -93,6 +74,22 @@ impl VTable for VarBin { 1 } + fn validate(&self, data: &VarBinData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.len() == len, + "VarBinArray length {} does not match outer length {}", + data.len(), + len + ); + vortex_ensure!( + data.dtype() == *dtype, + "VarBinArray dtype {} does not match outer dtype {}", + data.dtype(), + dtype + ); + Ok(()) + } + fn buffer(array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { match idx { 0 => array.bytes_handle().clone(), @@ -107,36 +104,27 @@ impl VTable for VarBin { } } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Ok(ProstMetadata(VarBinMetadata { - offsets_ptype: PType::try_from(array.offsets().dtype()) - .vortex_expect("Must be a valid PType") as i32, - })) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(Some( + VarBinMetadata { + offsets_ptype: PType::try_from(array.offsets().dtype()) + .vortex_expect("Must be a valid PType") as i32, + } + .encode_to_vec(), + )) } fn deserialize( - bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(ProstMetadata(ProstMetadata::::deserialize( - bytes, - )?)) - } - - fn build( + &self, dtype: &DType, len: usize, - metadata: &Self::Metadata, + metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + let metadata = VarBinMetadata::decode(metadata)?; let validity = if children.len() == 1 { Validity::from(dtype.nullability()) } else if children.len() == 2 { diff --git a/vortex-array/src/arrays/varbinview/array.rs b/vortex-array/src/arrays/varbinview/array.rs index d7228c65ed0..fd61c204feb 100644 --- a/vortex-array/src/arrays/varbinview/array.rs +++ b/vortex-array/src/arrays/varbinview/array.rs @@ -17,6 +17,7 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::array::Array; +use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::VarBinView; @@ -26,7 +27,6 @@ use crate::builders::ArrayBuilder; use crate::builders::VarBinViewBuilder; use crate::dtype::DType; use crate::dtype::Nullability; -use crate::stats::ArrayStats; use crate::validity::Validity; /// The validity bitmap indicating which elements are non-null. @@ -96,13 +96,13 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; #[derive(Clone, Debug)] pub struct VarBinViewData { pub(super) slots: Vec>, - pub(super) dtype: DType, + pub(super) is_utf8: bool, + pub(super) nullability: Nullability, pub(super) buffers: Arc<[BufferHandle]>, pub(super) views: BufferHandle, - pub(super) stats_set: ArrayStats, } -pub struct VarBinViewArrayParts { +pub struct VarBinViewDataParts { pub dtype: DType, pub buffers: Arc<[BufferHandle]>, pub views: BufferHandle, @@ -110,6 +110,22 @@ pub struct VarBinViewArrayParts { } impl VarBinViewData { + fn dtype_parts(dtype: &DType) -> VortexResult<(bool, Nullability)> { + match dtype { + DType::Utf8(nullability) => Ok((true, *nullability)), + DType::Binary(nullability) => Ok((false, *nullability)), + _ => vortex_bail!(InvalidArgument: "invalid DType {dtype} for `VarBinViewArray`"), + } + } + + fn make_dtype(is_utf8: bool, nullability: Nullability) -> DType { + if is_utf8 { + DType::Utf8(nullability) + } else { + DType::Binary(nullability) + } + } + /// Build the slots vector for this array. pub(super) fn make_slots(validity: &Validity, len: usize) -> Vec> { vec![validity_to_child(validity, len)] @@ -263,12 +279,14 @@ impl VarBinViewData { ) -> Self { let len = views.len() / size_of::(); let slots = Self::make_slots(&validity, len); + let (is_utf8, nullability) = + Self::dtype_parts(&dtype).vortex_expect("VarBinViewArray dtype must be utf8 or binary"); Self { slots, - views, + is_utf8, + nullability, buffers, - dtype, - stats_set: Default::default(), + views, } } @@ -367,8 +385,8 @@ impl VarBinViewData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - &self.dtype + pub fn dtype(&self) -> DType { + Self::make_dtype(self.is_utf8, self.nullability) } /// Returns `true` if this array is empty. @@ -379,7 +397,7 @@ impl VarBinViewData { /// Returns the [`Validity`] of this array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) } /// Returns the validity as a [`Mask`]. @@ -388,10 +406,10 @@ impl VarBinViewData { } /// Splits the array into owned parts - pub fn into_parts(self) -> VarBinViewArrayParts { + pub fn into_parts(self) -> VarBinViewDataParts { let validity = self.validity(); - VarBinViewArrayParts { - dtype: self.dtype, + VarBinViewDataParts { + dtype: self.dtype(), buffers: self.buffers, views: self.views, validity, @@ -552,6 +570,13 @@ impl VarBinViewData { } impl Array { + #[inline] + fn from_prevalidated_data(data: VarBinViewData) -> Self { + let dtype = data.dtype(); + let len = data.len(); + unsafe { Array::from_parts_unchecked(ArrayParts::new(VarBinView, dtype, len, data)) } + } + /// Construct a `VarBinViewArray` from an iterator of optional byte slices. #[expect( clippy::same_name_method, @@ -561,32 +586,27 @@ impl Array { iter: I, dtype: DType, ) -> Self { - Array::try_from_data(VarBinViewData::from_iter(iter, dtype)) - .vortex_expect("VarBinViewData is always valid") + Self::from_prevalidated_data(VarBinViewData::from_iter(iter, dtype)) } pub fn from_iter_str, I: IntoIterator>(iter: I) -> Self { - Array::try_from_data(VarBinViewData::from_iter_str(iter)) - .vortex_expect("VarBinViewData is always valid") + Self::from_prevalidated_data(VarBinViewData::from_iter_str(iter)) } pub fn from_iter_nullable_str, I: IntoIterator>>( iter: I, ) -> Self { - Array::try_from_data(VarBinViewData::from_iter_nullable_str(iter)) - .vortex_expect("VarBinViewData is always valid") + Self::from_prevalidated_data(VarBinViewData::from_iter_nullable_str(iter)) } pub fn from_iter_bin, I: IntoIterator>(iter: I) -> Self { - Array::try_from_data(VarBinViewData::from_iter_bin(iter)) - .vortex_expect("VarBinViewData is always valid") + Self::from_prevalidated_data(VarBinViewData::from_iter_bin(iter)) } pub fn from_iter_nullable_bin, I: IntoIterator>>( iter: I, ) -> Self { - Array::try_from_data(VarBinViewData::from_iter_nullable_bin(iter)) - .vortex_expect("VarBinViewData is always valid") + Self::from_prevalidated_data(VarBinViewData::from_iter_nullable_bin(iter)) } /// Creates a new `VarBinViewArray`. @@ -596,7 +616,9 @@ impl Array { dtype: DType, validity: Validity, ) -> VortexResult { - Array::try_from_data(VarBinViewData::try_new(views, buffers, dtype, validity)?) + Ok(Self::from_prevalidated_data(VarBinViewData::try_new( + views, buffers, dtype, validity, + )?)) } /// Creates a new `VarBinViewArray` without validation. @@ -610,10 +632,9 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Array::try_from_data(unsafe { + Self::from_prevalidated_data(unsafe { VarBinViewData::new_unchecked(views, buffers, dtype, validity) }) - .vortex_expect("VarBinViewData is always valid") } /// Creates a new `VarBinViewArray` with device or host memory. @@ -623,8 +644,7 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Array::try_from_data(VarBinViewData::new_handle(views, buffers, dtype, validity)) - .vortex_expect("VarBinViewData is always valid") + Self::from_prevalidated_data(VarBinViewData::new_handle(views, buffers, dtype, validity)) } /// Construct a new array from `BufferHandle`s without validation. @@ -638,10 +658,9 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Array::try_from_data(unsafe { + Self::from_prevalidated_data(unsafe { VarBinViewData::new_handle_unchecked(views, buffers, dtype, validity) }) - .vortex_expect("VarBinViewData is always valid") } } @@ -673,28 +692,24 @@ impl<'a> FromIterator> for VarBinViewData { impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Array::try_from_data(>::from_iter(iter)) - .vortex_expect(" is always valid") + Self::from_prevalidated_data(>::from_iter(iter)) } } impl FromIterator>> for Array { fn from_iter>>>(iter: T) -> Self { - Array::try_from_data(>::from_iter(iter)) - .vortex_expect(" is always valid") + Self::from_prevalidated_data(>::from_iter(iter)) } } impl FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Array::try_from_data(>::from_iter(iter)) - .vortex_expect(" is always valid") + Self::from_prevalidated_data(>::from_iter(iter)) } } impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Array::try_from_data(>::from_iter(iter)) - .vortex_expect(" is always valid") + Self::from_prevalidated_data(>::from_iter(iter)) } } diff --git a/vortex-array/src/arrays/varbinview/mod.rs b/vortex-array/src/arrays/varbinview/mod.rs index d29fd5bf3e5..d2d0041424d 100644 --- a/vortex-array/src/arrays/varbinview/mod.rs +++ b/vortex-array/src/arrays/varbinview/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::VarBinViewArrayParts; pub use array::VarBinViewData; +pub use array::VarBinViewDataParts; pub use vtable::VarBinViewArray; mod accessor; diff --git a/vortex-array/src/arrays/varbinview/vtable/mod.rs b/vortex-array/src/arrays/varbinview/vtable/mod.rs index 2efb586e619..7c2b674aa9e 100644 --- a/vortex-array/src/arrays/varbinview/vtable/mod.rs +++ b/vortex-array/src/arrays/varbinview/vtable/mod.rs @@ -14,7 +14,6 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; -use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; @@ -32,7 +31,6 @@ use crate::dtype::DType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; mod kernel; @@ -50,29 +48,13 @@ impl VarBinView { impl VTable for VarBinView { type ArrayData = VarBinViewData; - type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &VarBinViewData) -> &Self { - &VarBinView - } fn id(&self) -> ArrayId { Self::ID } - fn len(array: &VarBinViewData) -> usize { - array.views_handle().len() / size_of::() - } - - fn dtype(array: &VarBinViewData) -> &DType { - &array.dtype - } - - fn stats(array: &VarBinViewData) -> &ArrayStats { - &array.stats_set - } - fn array_hash( array: &VarBinViewData, state: &mut H, @@ -100,6 +82,22 @@ impl VTable for VarBinView { array.data_buffers().len() + 1 } + fn validate(&self, data: &VarBinViewData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + data.len() == len, + "VarBinViewArray length {} does not match outer length {}", + data.len(), + len + ); + vortex_ensure!( + data.dtype() == *dtype, + "VarBinViewArray dtype {} does not match outer dtype {}", + data.dtype(), + dtype + ); + Ok(()) + } + fn buffer(array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { let ndata = array.data_buffers().len(); if idx < ndata { @@ -122,31 +120,26 @@ impl VTable for VarBinView { } } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) - } - - fn build( + &self, dtype: &DType, len: usize, - _metadata: &Self::Metadata, + metadata: &[u8], + buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &VortexSession, ) -> VortexResult { + if !metadata.is_empty() { + vortex_bail!( + "VarBinViewArray expects empty metadata, got {} bytes", + metadata.len() + ); + } let Some((views_handle, data_handles)) = buffers.split_last() else { vortex_bail!("Expected at least 1 buffer, got 0"); }; @@ -242,8 +235,8 @@ mod tests { use crate::IntoArray; use crate::LEGACY_SESSION; use crate::assert_arrays_eq; - use crate::serde::ArrayParts; use crate::serde::SerializeOptions; + use crate::serde::SerializedArray; #[test] fn test_nullable_varbinview_serde_roundtrip() { @@ -268,7 +261,7 @@ mod tests { for buf in serialized { concat.extend_from_slice(buf.as_ref()); } - let parts = ArrayParts::try_from(concat.freeze()).unwrap(); + let parts = SerializedArray::try_from(concat.freeze()).unwrap(); let decoded = parts .decode( &dtype, diff --git a/vortex-array/src/arrays/variant/mod.rs b/vortex-array/src/arrays/variant/mod.rs index 29fdb5edcd2..02a89fae953 100644 --- a/vortex-array/src/arrays/variant/mod.rs +++ b/vortex-array/src/arrays/variant/mod.rs @@ -9,8 +9,8 @@ pub use self::vtable::Variant; pub use self::vtable::VariantArray; use crate::ArrayRef; use crate::array::Array; +use crate::array::ArrayParts; use crate::dtype::DType; -use crate::stats::ArrayStats; pub(super) const NUM_SLOTS: usize = 1; pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child"]; @@ -21,21 +21,18 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child"]; /// (e.g. a `ParquetVariantArray` or any other variant encoding). /// /// Nullability is delegated to the child array: `VariantArray`'s dtype is -/// always the child's dtype. The child's validity determines which rows are -/// null. +/// always `DType::Variant(child.dtype().nullability())`. The child's validity +/// determines which rows are null. #[derive(Clone, Debug)] pub struct VariantData { pub(super) slots: Vec>, - pub(crate) stats_set: ArrayStats, } impl VariantData { /// Creates a new VariantArray. Nullability comes from the child's dtype. pub fn new(child: ArrayRef) -> Self { - let stats_set = child.statistics().to_array_stats(); Self { slots: vec![Some(child)], - stats_set, } } @@ -45,8 +42,8 @@ impl VariantData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> &DType { - self.child().dtype() + pub fn dtype(&self) -> DType { + DType::Variant(self.child().dtype().nullability()) } /// Returns `true` if this array is empty. @@ -65,6 +62,17 @@ impl VariantData { impl Array { /// Creates a new `VariantArray`. pub fn new(child: ArrayRef) -> Self { - Array::try_from_data(VariantData::new(child)).vortex_expect("VariantData is always valid") + let dtype = DType::Variant(child.dtype().nullability()); + let len = child.len(); + let stats = child.statistics().to_owned(); + unsafe { + Array::from_parts_unchecked(ArrayParts::new( + Variant, + dtype, + len, + VariantData::new(child), + )) + } + .with_stats_set(stats) } } diff --git a/vortex-array/src/arrays/variant/vtable/mod.rs b/vortex-array/src/arrays/variant/vtable/mod.rs index 31f52911e6d..49981a6f955 100644 --- a/vortex-array/src/arrays/variant/vtable/mod.rs +++ b/vortex-array/src/arrays/variant/vtable/mod.rs @@ -13,7 +13,6 @@ use vortex_error::vortex_panic; use crate::ArrayEq; use crate::ArrayHash; use crate::ArrayRef; -use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; @@ -27,7 +26,6 @@ use crate::arrays::variant::VariantData; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::serde::ArrayChildren; -use crate::stats::ArrayStats; use crate::vtable; vtable!(Variant, Variant, VariantData); @@ -42,30 +40,32 @@ impl Variant { impl VTable for Variant { type ArrayData = VariantData; - type Metadata = EmptyMetadata; - type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(_array: &Self::ArrayData) -> &Self { - &Variant - } - fn id(&self) -> ArrayId { Self::ID } - fn len(array: &Self::ArrayData) -> usize { - array.child().len() - } - - fn dtype(array: &Self::ArrayData) -> &DType { - array.child().dtype() - } - - fn stats(array: &Self::ArrayData) -> &ArrayStats { - &array.stats_set + fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!( + matches!(dtype, DType::Variant(_)), + "Expected Variant DType, got {dtype}" + ); + vortex_ensure!( + data.child().dtype() == dtype, + "VariantArray child dtype {} does not match outer dtype {}", + data.child().dtype(), + dtype + ); + vortex_ensure!( + data.len() == len, + "VariantArray length {} does not match outer length {}", + data.len(), + len + ); + Ok(()) } fn array_hash(array: &VariantData, state: &mut H, precision: Precision) { @@ -88,31 +88,25 @@ impl VTable for Variant { None } - fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { - Ok(EmptyMetadata) - } - - fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - _bytes: &[u8], - _dtype: &DType, - _len: usize, - _buffers: &[BufferHandle], - _session: &vortex_session::VortexSession, - ) -> VortexResult { - Ok(EmptyMetadata) - } - - fn build( + &self, dtype: &DType, len: usize, - _metadata: &Self::Metadata, + metadata: &[u8], + _buffers: &[BufferHandle], children: &dyn ArrayChildren, + _session: &vortex_session::VortexSession, ) -> VortexResult { + vortex_ensure!( + metadata.is_empty(), + "VariantArray expects empty metadata, got {} bytes", + metadata.len() + ); vortex_ensure!(matches!(dtype, DType::Variant(_)), "Expected Variant DType"); vortex_ensure!( children.len() == 1, @@ -159,11 +153,16 @@ impl VTable for Variant { mod tests { use super::*; use crate::IntoArray; - use crate::arrays::PrimitiveArray; + use crate::arrays::ConstantArray; + use crate::dtype::DType; + use crate::dtype::Nullability; + use crate::scalar::Scalar; #[test] fn with_slots_rejects_missing_child() { - let array = VariantArray::new(PrimitiveArray::from_iter([1u8, 2, 3]).into_array()); + let child = + ConstantArray::new(Scalar::null(DType::Variant(Nullability::Nullable)), 3).into_array(); + let array = VariantArray::new(child); let mut data = array.into_data(); let err = ::with_slots(&mut data, vec![None]).unwrap_err(); diff --git a/vortex-array/src/arrow/executor/dictionary.rs b/vortex-array/src/arrow/executor/dictionary.rs index 5d9b75182b3..23cc71e65dd 100644 --- a/vortex-array/src/arrow/executor/dictionary.rs +++ b/vortex-array/src/arrow/executor/dictionary.rs @@ -21,7 +21,7 @@ use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::Dict; use crate::arrays::DictArray; -use crate::arrays::dict::DictArrayParts; +use crate::arrays::dict::DictDataParts; use crate::arrow::ArrowArrayExecutor; pub(super) fn to_arrow_dictionary( @@ -78,7 +78,7 @@ fn dict_to_dict( values_type: &DataType, ctx: &mut ExecutionCtx, ) -> VortexResult { - let DictArrayParts { codes, values, .. } = array.into_data().into_parts(); + let DictDataParts { codes, values, .. } = array.into_data().into_parts(); let codes = codes.execute_arrow(Some(codes_type), ctx)?; let values = values.execute_arrow(Some(values_type), ctx)?; make_dict_array(codes_type, codes, values) diff --git a/vortex-array/src/arrow/executor/list.rs b/vortex-array/src/arrow/executor/list.rs index 661e459f62e..f17ededa54d 100644 --- a/vortex-array/src/arrow/executor/list.rs +++ b/vortex-array/src/arrow/executor/list.rs @@ -21,7 +21,7 @@ use crate::arrays::List; use crate::arrays::ListArray; use crate::arrays::ListView; use crate::arrays::ListViewArray; -use crate::arrays::listview::ListViewArrayParts; +use crate::arrays::listview::ListViewDataParts; use crate::arrays::listview::ListViewRebuildMode; use crate::arrow::ArrowArrayExecutor; use crate::arrow::executor::validity::to_arrow_null_buffer; @@ -133,7 +133,7 @@ fn list_view_zctl( ))); } - let ListViewArrayParts { + let ListViewDataParts { elements, offsets, sizes, diff --git a/vortex-array/src/arrow/executor/list_view.rs b/vortex-array/src/arrow/executor/list_view.rs index fb536bc3c1b..614ea6e7c93 100644 --- a/vortex-array/src/arrow/executor/list_view.rs +++ b/vortex-array/src/arrow/executor/list_view.rs @@ -14,7 +14,7 @@ use crate::ExecutionCtx; use crate::arrays::ListView; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; -use crate::arrays::listview::ListViewArrayParts; +use crate::arrays::listview::ListViewDataParts; use crate::arrow::ArrowArrayExecutor; use crate::arrow::executor::validity::to_arrow_null_buffer; use crate::builtins::ArrayBuiltins; @@ -43,7 +43,7 @@ fn list_view_to_list_view( elements_field: &FieldRef, ctx: &mut ExecutionCtx, ) -> VortexResult { - let ListViewArrayParts { + let ListViewDataParts { elements, offsets, sizes, diff --git a/vortex-array/src/arrow/executor/struct_.rs b/vortex-array/src/arrow/executor/struct_.rs index eab871de2eb..7edd1b05e35 100644 --- a/vortex-array/src/arrow/executor/struct_.rs +++ b/vortex-array/src/arrow/executor/struct_.rs @@ -19,7 +19,7 @@ use crate::arrays::Chunked; use crate::arrays::ScalarFnVTable; use crate::arrays::Struct; use crate::arrays::StructArray; -use crate::arrays::struct_::StructArrayParts; +use crate::arrays::struct_::StructDataParts; use crate::arrow::ArrowArrayExecutor; use crate::arrow::executor::validity::to_arrow_null_buffer; use crate::builtins::ArrayBuiltins; @@ -49,13 +49,14 @@ pub(super) fn to_arrow_struct( // Attempt to short-circuit if the array is already a Struct: let array = match array.try_into::() { Ok(array) => { - let len = array.len(); - let StructArrayParts { + let parts = array.into_parts(); + let len = parts.len; + let StructDataParts { validity, fields, struct_fields, .. - } = array.into_parts(); + } = parts.data.into_parts(); let validity = to_arrow_null_buffer(validity, len, ctx)?; return create_from_fields( target_fields.ok_or_else(|| struct_fields.names().clone()), @@ -97,13 +98,14 @@ pub(super) fn to_arrow_struct( }; let struct_array = array.execute::(ctx)?; - let len = struct_array.len(); - let StructArrayParts { + let parts = struct_array.into_parts(); + let len = parts.len; + let StructDataParts { validity, fields, struct_fields, .. - } = struct_array.into_parts(); + } = parts.data.into_parts(); let validity = to_arrow_null_buffer(validity, len, ctx)?; create_from_fields( diff --git a/vortex-array/src/canonical.rs b/vortex-array/src/canonical.rs index e1220bf4837..e50458bcebc 100644 --- a/vortex-array/src/canonical.rs +++ b/vortex-array/src/canonical.rs @@ -37,13 +37,13 @@ use crate::arrays::VarBinView; use crate::arrays::VarBinViewArray; use crate::arrays::Variant; use crate::arrays::VariantArray; -use crate::arrays::bool::BoolArrayParts; -use crate::arrays::decimal::DecimalArrayParts; -use crate::arrays::listview::ListViewArrayParts; +use crate::arrays::bool::BoolDataParts; +use crate::arrays::decimal::DecimalDataParts; +use crate::arrays::listview::ListViewDataParts; use crate::arrays::listview::ListViewRebuildMode; -use crate::arrays::primitive::PrimitiveArrayParts; -use crate::arrays::struct_::StructArrayParts; -use crate::arrays::varbinview::VarBinViewArrayParts; +use crate::arrays::primitive::PrimitiveDataParts; +use crate::arrays::struct_::StructDataParts; +use crate::arrays::varbinview::VarBinViewDataParts; use crate::dtype::DType; use crate::dtype::NativePType; use crate::dtype::Nullability; @@ -544,18 +544,19 @@ impl Executable for CanonicalValidity { match array.execute::(ctx)? { n @ Canonical::Null(_) => Ok(CanonicalValidity(n)), Canonical::Bool(b) => { - let BoolArrayParts { + let b = b.into_parts(); + let BoolDataParts { bits, offset, len, validity, - } = b.into_parts(); + } = b.data.into_parts(); Ok(CanonicalValidity(Canonical::Bool( BoolArray::try_new_from_handle(bits, offset, len, validity.execute(ctx)?)?, ))) } Canonical::Primitive(p) => { - let PrimitiveArrayParts { + let PrimitiveDataParts { ptype, buffer, validity, @@ -565,7 +566,7 @@ impl Executable for CanonicalValidity { }))) } Canonical::Decimal(d) => { - let DecimalArrayParts { + let DecimalDataParts { decimal_dtype, values, values_type, @@ -581,7 +582,7 @@ impl Executable for CanonicalValidity { }))) } Canonical::VarBinView(vbv) => { - let VarBinViewArrayParts { + let VarBinViewDataParts { dtype, buffers, views, @@ -598,7 +599,7 @@ impl Executable for CanonicalValidity { } Canonical::List(l) => { let zctl = l.is_zero_copy_to_list(); - let ListViewArrayParts { + let ListViewDataParts { elements, offsets, sizes, @@ -619,12 +620,13 @@ impl Executable for CanonicalValidity { ))) } Canonical::Struct(st) => { - let len = st.len(); - let StructArrayParts { + let parts = st.into_parts(); + let len = parts.len; + let StructDataParts { struct_fields, fields, validity, - } = st.into_parts(); + } = parts.data.into_parts(); Ok(CanonicalValidity(Canonical::Struct(unsafe { StructArray::new_unchecked(fields, struct_fields, len, validity.execute(ctx)?) }))) @@ -664,18 +666,19 @@ impl Executable for RecursiveCanonical { match array.execute::(ctx)? { n @ Canonical::Null(_) => Ok(RecursiveCanonical(n)), Canonical::Bool(b) => { - let BoolArrayParts { + let b = b.into_parts(); + let BoolDataParts { bits, offset, len, validity, - } = b.into_parts(); + } = b.data.into_parts(); Ok(RecursiveCanonical(Canonical::Bool( BoolArray::try_new_from_handle(bits, offset, len, validity.execute(ctx)?)?, ))) } Canonical::Primitive(p) => { - let PrimitiveArrayParts { + let PrimitiveDataParts { ptype, buffer, validity, @@ -685,7 +688,7 @@ impl Executable for RecursiveCanonical { }))) } Canonical::Decimal(d) => { - let DecimalArrayParts { + let DecimalDataParts { decimal_dtype, values, values_type, @@ -701,7 +704,7 @@ impl Executable for RecursiveCanonical { }))) } Canonical::VarBinView(vbv) => { - let VarBinViewArrayParts { + let VarBinViewDataParts { dtype, buffers, views, @@ -718,7 +721,7 @@ impl Executable for RecursiveCanonical { } Canonical::List(l) => { let zctl = l.is_zero_copy_to_list(); - let ListViewArrayParts { + let ListViewDataParts { elements, offsets, sizes, @@ -749,12 +752,13 @@ impl Executable for RecursiveCanonical { ))) } Canonical::Struct(st) => { - let len = st.len(); - let StructArrayParts { + let parts = st.into_parts(); + let len = parts.len; + let StructDataParts { struct_fields, fields, validity, - } = st.into_parts(); + } = parts.data.into_parts(); let executed_fields = fields .iter() .map(|f| Ok(f.clone().execute::(ctx)?.0.into_array())) diff --git a/vortex-array/src/display/extractors/stats.rs b/vortex-array/src/display/extractors/stats.rs index 70640e9619d..7f8fce7feb1 100644 --- a/vortex-array/src/display/extractors/stats.rs +++ b/vortex-array/src/display/extractors/stats.rs @@ -9,6 +9,7 @@ use crate::display::extractor::TreeContext; use crate::display::extractor::TreeExtractor; use crate::expr::stats::Stat; use crate::expr::stats::StatsProvider; +use crate::validity::Validity; /// Display wrapper for array statistics in compact format. /// @@ -39,16 +40,17 @@ impl fmt::Display for StatsDisplay<'_> { write!(f, "nulls={}", nc)?; } } else if self.0.dtype().is_nullable() { - match self.0.all_valid() { - Ok(true) => { + match self.0.validity() { + Ok(Validity::NonNullable | Validity::AllValid) => { sep(f)?; f.write_str("all_valid")?; } - Ok(false) => { - if self.0.all_invalid().unwrap_or(false) { - sep(f)?; - f.write_str("all_invalid")?; - } + Ok(Validity::AllInvalid) => { + sep(f)?; + f.write_str("all_invalid")?; + } + Ok(Validity::Array(_)) => { + // Avoid computing validity-array stats as a side effect of display. } Err(e) => { tracing::warn!("Failed to check validity: {e}"); diff --git a/vortex-array/src/display/mod.rs b/vortex-array/src/display/mod.rs index 79afebe0fbe..42d9ace73d1 100644 --- a/vortex-array/src/display/mod.rs +++ b/vortex-array/src/display/mod.rs @@ -649,6 +649,7 @@ mod test { use crate::IntoArray as _; use crate::arrays::BoolArray; use crate::arrays::ListArray; + use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; use crate::display::DISPLAY_LIMIT; use crate::dtype::FieldNames; @@ -665,10 +666,8 @@ mod test { let x = buffer![1, 2, 3, 4].into_array(); assert_eq!(x.display_values().to_string(), "[1i32, 2i32, 3i32, 4i32]"); - let x = crate::arrays::PrimitiveArray::from_iter( - 0i32..i32::try_from(DISPLAY_LIMIT).unwrap() + 1, - ) - .into_array(); + let x = + PrimitiveArray::from_iter(0i32..i32::try_from(DISPLAY_LIMIT).unwrap() + 1).into_array(); assert_eq!( x.display_values().to_string(), "[0i32, 1i32, 2i32, 3i32, 4i32, 5i32, 6i32, 7i32, 8i32, 9i32, 10i32, 11i32, 12i32, ..., 14i32, 15i32, 16i32]" @@ -717,6 +716,14 @@ mod test { ); } + #[test] + fn test_display_tree_nullable_primitive_validity_child() { + let array = + PrimitiveArray::from_option_iter([Some(1i64), Some(2), None, Some(3)]).into_array(); + let expected = "root: vortex.primitive(i64?, len=4) nbytes=33 B (100.00%)\n metadata: EmptyMetadata\n buffer: values host 32 B (align=8) (96.97%)\n validity: vortex.bool(bool, len=4) nbytes=1 B (3.03%)\n metadata: BoolMetadata { offset: 0 }\n buffer: bits host 1 B (align=1) (100.00%)\n"; + assert_eq!(format!("{}", array.display_tree()), expected); + } + #[test] fn test_table_display_primitive() { use crate::display::DisplayOptions; @@ -743,13 +750,8 @@ mod test { fn test_table_display() { use crate::display::DisplayOptions; - let array = crate::arrays::PrimitiveArray::from_option_iter(vec![ - Some(-1), - Some(-2), - Some(-3), - None, - ]) - .into_array(); + let array = + PrimitiveArray::from_option_iter(vec![Some(-1), Some(-2), Some(-3), None]).into_array(); let struct_ = StructArray::try_from_iter_with_validity( [("x", buffer![1, 2, 3, 4].into_array()), ("y", array)], diff --git a/vortex-array/src/serde.rs b/vortex-array/src/serde.rs index b98f1da6c38..6f296ac3885 100644 --- a/vortex-array/src/serde.rs +++ b/vortex-array/src/serde.rs @@ -281,14 +281,14 @@ impl ArrayChildren for &[ArrayRef] { } } -/// [`ArrayParts`] represents a parsed but not-yet-decoded deserialized array. +/// [`SerializedArray`] represents a parsed but not-yet-decoded deserialized array. /// It contains all the information from the serialized form, without anything extra. i.e. /// it is missing a [`DType`] and `len`, and the `encoding_id` is not yet resolved to a concrete /// vtable. /// -/// An [`ArrayParts`] can be fully decoded into an [`ArrayRef`] using the `decode` function. +/// An [`SerializedArray`] can be fully decoded into an [`ArrayRef`] using the `decode` function. #[derive(Clone)] -pub struct ArrayParts { +pub struct SerializedArray { // Typed as fb::ArrayNode flatbuffer: FlatBuffer, // The location of the current fb::ArrayNode @@ -296,9 +296,9 @@ pub struct ArrayParts { buffers: Arc<[BufferHandle]>, } -impl Debug for ArrayParts { +impl Debug for SerializedArray { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ArrayParts") + f.debug_struct("SerializedArray") .field("encoding_id", &self.encoding_id()) .field("children", &(0..self.nchildren()).map(|i| self.child(i))) .field( @@ -310,8 +310,8 @@ impl Debug for ArrayParts { } } -impl ArrayParts { - /// Decode an [`ArrayParts`] into an [`ArrayRef`]. +impl SerializedArray { + /// Decode an [`SerializedArray`] into an [`ArrayRef`]. pub fn decode( &self, dtype: &DType, @@ -329,8 +329,8 @@ impl ArrayParts { .find(&encoding_id) .ok_or_else(|| vortex_err!("Unknown encoding: {}", encoding_id))?; - let children = ArrayPartsChildren { - parts: self, + let children = SerializedArrayChildren { + ser: self, ctx, session, }; @@ -402,7 +402,7 @@ impl ArrayParts { } /// Returns the nth child of the array. - pub fn child(&self, idx: usize) -> ArrayParts { + pub fn child(&self, idx: usize) -> SerializedArray { let children = self .flatbuffer() .children() @@ -487,7 +487,7 @@ impl ArrayParts { /// without needing to access the actual buffer data. pub fn buffer_lengths(&self) -> Vec { let fb_array = root::(self.flatbuffer.as_ref()) - .vortex_expect("ArrayParts flatbuffer must be a valid Array"); + .vortex_expect("SerializedArray flatbuffer must be a valid Array"); fb_array .buffers() .map(|buffers| buffers.iter().map(|b| b.length() as usize).collect()) @@ -505,7 +505,7 @@ impl ArrayParts { Ok((fb_buffer, flatbuffer_loc)) } - /// Create an [`ArrayParts`] from a pre-existing array tree flatbuffer and pre-resolved buffer + /// Create an [`SerializedArray`] from a pre-existing array tree flatbuffer and pre-resolved buffer /// handles. /// /// The caller is responsible for resolving buffers from whatever source (device segments, host @@ -516,24 +516,24 @@ impl ArrayParts { buffers: Vec, ) -> VortexResult { let (flatbuffer, flatbuffer_loc) = Self::validate_array_tree(array_tree)?; - Ok(ArrayParts { + Ok(SerializedArray { flatbuffer, flatbuffer_loc, buffers: buffers.into(), }) } - /// Create an [`ArrayParts`] from a raw array tree flatbuffer (metadata only). + /// Create an [`SerializedArray`] from a raw array tree flatbuffer (metadata only). /// - /// This constructor creates an `ArrayParts` with no buffer data, useful for + /// This constructor creates a `SerializedArray` with no buffer data, useful for /// inspecting the metadata when the actual buffer data is not needed /// (e.g., displaying buffer sizes from inlined array tree metadata). /// - /// Note: Calling `buffer()` on the returned `ArrayParts` will fail since + /// Note: Calling `buffer()` on the returned `SerializedArray` will fail since /// no actual buffer data is available. pub fn from_array_tree(array_tree: impl Into) -> VortexResult { let (flatbuffer, flatbuffer_loc) = Self::validate_array_tree(array_tree)?; - Ok(ArrayParts { + Ok(SerializedArray { flatbuffer, flatbuffer_loc, buffers: Arc::new([]), @@ -545,7 +545,7 @@ impl ArrayParts { unsafe { fba::ArrayNode::follow(self.flatbuffer.as_ref(), self.flatbuffer_loc) } } - /// Returns a new [`ArrayParts`] with the given node as the root + /// Returns a new [`SerializedArray`] with the given node as the root // TODO(ngates): we may want a wrapper that avoids this clone. fn with_root(&self, root: fba::ArrayNode) -> Self { let mut this = self.clone(); @@ -553,7 +553,7 @@ impl ArrayParts { this } - /// Create an [`ArrayParts`] from a pre-existing flatbuffer (ArrayNode) and a segment containing + /// Create an [`SerializedArray`] from a pre-existing flatbuffer (ArrayNode) and a segment containing /// only the data buffers (without the flatbuffer suffix). /// /// This is used when the flatbuffer is stored separately in layout metadata (e.g., when @@ -566,7 +566,7 @@ impl ArrayParts { Self::from_flatbuffer_and_segment_with_overrides(array_tree, segment, &HashMap::new()) } - /// Create an [`ArrayParts`] from a pre-existing flatbuffer (ArrayNode) and a segment, + /// Create an [`SerializedArray`] from a pre-existing flatbuffer (ArrayNode) and a segment, /// substituting host-resident buffer overrides for specific buffer indices. /// /// Buffers whose index appears in `buffer_overrides` are resolved from the provided @@ -611,7 +611,7 @@ impl ArrayParts { }) .collect::>>()?; - Ok(ArrayParts { + Ok(SerializedArray { flatbuffer: fb_buffer, flatbuffer_loc, buffers, @@ -619,31 +619,31 @@ impl ArrayParts { } } -struct ArrayPartsChildren<'a> { - parts: &'a ArrayParts, +struct SerializedArrayChildren<'a> { + ser: &'a SerializedArray, ctx: &'a ReadContext, session: &'a VortexSession, } -impl ArrayChildren for ArrayPartsChildren<'_> { +impl ArrayChildren for SerializedArrayChildren<'_> { fn get(&self, index: usize, dtype: &DType, len: usize) -> VortexResult { - self.parts + self.ser .child(index) .decode(dtype, len, self.ctx, self.session) } fn len(&self) -> usize { - self.parts.nchildren() + self.ser.nchildren() } } -impl TryFrom for ArrayParts { +impl TryFrom for SerializedArray { type Error = VortexError; fn try_from(value: ByteBuffer) -> Result { // The final 4 bytes contain the length of the flatbuffer. if value.len() < 4 { - vortex_bail!("ArrayParts buffer is too short"); + vortex_bail!("SerializedArray buffer is too short"); } // We align each buffer individually, so we remove alignment requirements on the buffer. @@ -651,7 +651,7 @@ impl TryFrom for ArrayParts { let fb_length = u32::try_from_le_bytes(&value.as_slice()[value.len() - 4..])? as usize; if value.len() < 4 + fb_length { - vortex_bail!("ArrayParts buffer is too short for flatbuffer"); + vortex_bail!("SerializedArray buffer is too short for flatbuffer"); } let fb_offset = value.len() - 4 - fb_length; @@ -662,7 +662,7 @@ impl TryFrom for ArrayParts { } } -impl TryFrom for ArrayParts { +impl TryFrom for SerializedArray { type Error = VortexError; fn try_from(value: BufferHandle) -> Result { diff --git a/vortex-array/src/stats/array.rs b/vortex-array/src/stats/array.rs index b3838bd4f9e..bd2e3cf7dea 100644 --- a/vortex-array/src/stats/array.rs +++ b/vortex-array/src/stats/array.rs @@ -83,6 +83,10 @@ impl From for StatsSet { } impl StatsSetRef<'_> { + pub(crate) fn replace(&self, stats: StatsSet) { + *self.array_stats.inner.write() = stats; + } + pub fn set_iter(&self, iter: StatsSetIntoIter) { let mut guard = self.array_stats.inner.write(); for (stat, value) in iter { diff --git a/vortex-array/src/test_harness.rs b/vortex-array/src/test_harness.rs index 348f79829fd..cee718f3920 100644 --- a/vortex-array/src/test_harness.rs +++ b/vortex-array/src/test_harness.rs @@ -8,25 +8,18 @@ use goldenfile::differs::binary_diff; use itertools::Itertools; use vortex_error::VortexResult; -use crate::DeserializeMetadata; -use crate::SerializeMetadata; use crate::arrays::BoolArray; /// Check that a named metadata matches its previous versioning. /// /// Goldenfile takes care of checking for equality against a checked-in file. #[allow(clippy::unwrap_used)] -pub fn check_metadata(name: &str, metadata: T) -where - T: SerializeMetadata, - T: DeserializeMetadata, -{ +pub fn check_metadata(name: &str, metadata: &[u8]) { let mut mint = Mint::new("goldenfiles/"); - let meta = metadata.serialize(); let mut f = mint .new_goldenfile_with_differ(name, Box::new(binary_diff)) .unwrap(); - f.write_all(&meta).unwrap(); + f.write_all(metadata).unwrap(); } /// Outputs the indices of the true values in a BoolArray diff --git a/vortex-btrblocks/src/schemes/float.rs b/vortex-btrblocks/src/schemes/float.rs index b995f0827c5..3ed6a883c44 100644 --- a/vortex-btrblocks/src/schemes/float.rs +++ b/vortex-btrblocks/src/schemes/float.rs @@ -6,7 +6,6 @@ use vortex_alp::ALP; use vortex_alp::RDEncoder; use vortex_alp::alp_encode; -use vortex_array::Array; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; @@ -147,6 +146,8 @@ impl Scheme for ALPRDScheme { }; let alp_rd = encoder.encode(stats.source()); + let dtype = alp_rd.dtype().clone(); + let right_bit_width = alp_rd.right_bit_width(); let mut alp_rd_data = alp_rd.into_data(); let patches = alp_rd_data @@ -154,8 +155,17 @@ impl Scheme for ALPRDScheme { .map(compress_patches) .transpose()?; alp_rd_data.replace_left_parts_patches(patches); - - Ok(Array::::try_from_data(alp_rd_data)?.into_array()) + let parts = alp_rd_data.into_parts(); + + Ok(vortex_alp::ALPRD::try_new( + dtype, + parts.left_parts, + parts.left_parts_dictionary, + parts.right_parts, + right_bit_width, + parts.left_parts_patches, + )? + .into_array()) } } diff --git a/vortex-btrblocks/src/schemes/integer.rs b/vortex-btrblocks/src/schemes/integer.rs index c6b4f3fee56..9309637c158 100644 --- a/vortex-btrblocks/src/schemes/integer.rs +++ b/vortex-btrblocks/src/schemes/integer.rs @@ -3,7 +3,6 @@ //! Integer compression schemes. -use vortex_array::Array; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; @@ -336,12 +335,27 @@ impl Scheme for BitPackingScheme { return Ok(stats.source().clone().into_array()); } let packed = bitpack_encode(stats.source(), bw, Some(&histogram))?; + let packed_stats = packed.statistics().to_owned(); + let ptype = packed.dtype().as_ptype(); + let len = packed.len(); + let nullability = packed.dtype().nullability(); let mut packed_data = packed.into_data(); - let patches = packed_data.patches().map(compress_patches).transpose()?; + let patches = packed_data.patches(len).map(compress_patches).transpose()?; packed_data.replace_patches(patches); - - Ok(Array::::try_from_data(packed_data)?.into_array()) + let parts = packed_data.into_parts(len, nullability); + + Ok(vortex_fastlanes::BitPacked::try_new( + parts.packed, + ptype, + parts.validity, + parts.patches, + parts.bit_width, + parts.len, + parts.offset, + )? + .with_stats_set(packed_stats) + .into_array()) } } @@ -857,6 +871,9 @@ mod scheme_selection_tests { use vortex_array::arrays::Constant; use vortex_array::arrays::Dict; use vortex_array::arrays::PrimitiveArray; + use vortex_array::expr::stats::Precision; + use vortex_array::expr::stats::Stat; + use vortex_array::expr::stats::StatsProviderExt; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexResult; @@ -895,6 +912,18 @@ mod scheme_selection_tests { let btr = BtrBlocksCompressor::default(); let compressed = btr.compress(&array.into_array())?; assert!(compressed.is::()); + assert_eq!( + compressed.statistics().get_as::(Stat::NullCount), + Some(Precision::exact(0u64)) + ); + assert_eq!( + compressed.statistics().get_as::(Stat::Min), + Some(Precision::exact(0u32)) + ); + assert_eq!( + compressed.statistics().get_as::(Stat::Max), + Some(Precision::exact(15u32)) + ); Ok(()) } diff --git a/vortex-btrblocks/src/schemes/rle.rs b/vortex-btrblocks/src/schemes/rle.rs index 4da83209f39..42fd4295bb7 100644 --- a/vortex-btrblocks/src/schemes/rle.rs +++ b/vortex-btrblocks/src/schemes/rle.rs @@ -21,6 +21,8 @@ use vortex_compressor::scheme::SchemeId; use vortex_compressor::stats::FloatStats; use vortex_compressor::stats::IntegerStats; use vortex_error::VortexResult; +#[cfg(feature = "unstable_encodings")] +use vortex_fastlanes::Delta; use vortex_fastlanes::RLE; use crate::ArrayAndStats; @@ -281,7 +283,6 @@ impl Scheme for RLEScheme { compressed_values, compressed_indices, compressed_offsets, - rle_array.dtype().clone(), rle_array.offset(), rle_array.len(), ) @@ -306,6 +307,5 @@ fn try_compress_delta( let compressed_deltas = compressor.compress_child(&deltas.into_array(), parent_ctx, parent_id, child_index)?; - vortex_fastlanes::DeltaData::try_new(compressed_bases, compressed_deltas, 0, child.len()) - .map(IntoArray::into_array) + Delta::try_new(compressed_bases, compressed_deltas, 0, child.len()).map(IntoArray::into_array) } diff --git a/vortex-btrblocks/src/schemes/string.rs b/vortex-btrblocks/src/schemes/string.rs index 06ba6d03314..f222ce1623b 100644 --- a/vortex-btrblocks/src/schemes/string.rs +++ b/vortex-btrblocks/src/schemes/string.rs @@ -240,7 +240,7 @@ impl Scheme for ZstdBuffersScheme { let stats = data.string_stats(); Ok( - vortex_zstd::ZstdBuffersData::compress(&stats.source().clone().into_array(), 3)? + vortex_zstd::ZstdBuffers::compress(&stats.source().clone().into_array(), 3)? .into_array(), ) } diff --git a/vortex-cuda/benches/date_time_parts_cuda.rs b/vortex-cuda/benches/date_time_parts_cuda.rs index a91111e3f8a..53f303d0b2c 100644 --- a/vortex-cuda/benches/date_time_parts_cuda.rs +++ b/vortex-cuda/benches/date_time_parts_cuda.rs @@ -24,8 +24,8 @@ use vortex::array::validity::Validity; use vortex::buffer::Buffer; use vortex::dtype::DType; use vortex::dtype::Nullability; +use vortex::encodings::datetime_parts::DateTimeParts; use vortex::encodings::datetime_parts::DateTimePartsArray; -use vortex::encodings::datetime_parts::DateTimePartsData; use vortex::error::VortexExpect; use vortex::extension::datetime::TimeUnit; use vortex::extension::datetime::Timestamp; @@ -45,11 +45,8 @@ fn make_datetimeparts_array(len: usize, time_unit: TimeUnit) -> DateTimePartsArr let dtype = DType::Extension(Timestamp::new(time_unit, Nullability::NonNullable).erased()); - DateTimePartsArray::try_from_data( - DateTimePartsData::try_new(dtype, days_arr, seconds_arr, subseconds_arr) - .vortex_expect("Failed to create DateTimePartsArray"), - ) - .vortex_expect("DateTimePartsData is always valid") + DateTimeParts::try_new(dtype, days_arr, seconds_arr, subseconds_arr) + .vortex_expect("Failed to create DateTimePartsArray") } fn benchmark_datetimeparts(c: &mut Criterion) { diff --git a/vortex-cuda/benches/dynamic_dispatch_cuda.rs b/vortex-cuda/benches/dynamic_dispatch_cuda.rs index 21537b51031..6d4bb42d619 100644 --- a/vortex-cuda/benches/dynamic_dispatch_cuda.rs +++ b/vortex-cuda/benches/dynamic_dispatch_cuda.rs @@ -30,6 +30,7 @@ use vortex::encodings::alp::ALPFloat; use vortex::encodings::alp::Exponents; use vortex::encodings::alp::alp_encode; use vortex::encodings::fastlanes::BitPackedData; +use vortex::encodings::fastlanes::FoR; use vortex::encodings::fastlanes::FoRData; use vortex::encodings::runend::RunEnd; use vortex::error::VortexExpect; @@ -187,9 +188,9 @@ fn bench_for_bitpacked(c: &mut Criterion) { .collect(); let prim = PrimitiveArray::new(Buffer::from(residuals), NonNullable); let bp = BitPackedData::encode(&prim.into_array(), bit_width).vortex_expect("bitpack"); - let for_arr = - FoRData::try_new(bp.into_array(), Scalar::from(reference)).vortex_expect("for"); - let array = for_arr.into_array(); + let array = FoR::try_new(bp.into_array(), Scalar::from(reference)) + .vortex_expect("for") + .into_array(); group.bench_with_input( BenchmarkId::new("dynamic_dispatch_u32", len_str), @@ -320,8 +321,8 @@ fn bench_dict_bp_codes_bp_for_values(c: &mut Criterion) { let dict_prim = PrimitiveArray::new(Buffer::from(dict_residuals), NonNullable); let dict_bp = BitPackedData::encode(&dict_prim.into_array(), dict_bit_width) .vortex_expect("bitpack dict"); - let dict_for = FoRData::try_new(dict_bp.into_array(), Scalar::from(dict_reference)) - .vortex_expect("for dict"); + let dict_for = + FoR::try_new(dict_bp.into_array(), Scalar::from(dict_reference)).vortex_expect("for dict"); for (len, len_str) in BENCH_ARGS { group.throughput(Throughput::Bytes((len * size_of::()) as u64)); @@ -384,7 +385,7 @@ fn bench_alp_for_bitpacked(c: &mut Criterion) { BitPackedData::encode(for_arr.encoded(), bit_width).vortex_expect("bitpack encode"); let tree = ALP::new( - FoRData::try_new(bp.into_array(), for_arr.reference_scalar().clone()) + FoR::try_new(bp.into_array(), for_arr.reference_scalar().clone()) .vortex_expect("for_new") .into_array(), exponents, diff --git a/vortex-cuda/benches/for_cuda.rs b/vortex-cuda/benches/for_cuda.rs index eb87eb6d770..cc5fab5e22c 100644 --- a/vortex-cuda/benches/for_cuda.rs +++ b/vortex-cuda/benches/for_cuda.rs @@ -26,8 +26,8 @@ use vortex::buffer::Buffer; use vortex::dtype::NativePType; use vortex::dtype::PType; use vortex::encodings::fastlanes::BitPackedData; +use vortex::encodings::fastlanes::FoR; use vortex::encodings::fastlanes::FoRArray; -use vortex::encodings::fastlanes::FoRData; use vortex::error::VortexExpect; use vortex::scalar::Scalar; use vortex::session::VortexSession; @@ -57,17 +57,10 @@ where if bp && T::PTYPE != PType::U8 { let child = BitPackedData::encode(&primitive_array, 8).vortex_expect("failed to bitpack"); - FoRArray::try_from_data( - FoRData::try_new(child.into_array(), reference.into()) - .vortex_expect("failed to create FoR array"), - ) - .vortex_expect("FoRData is always valid") + FoR::try_new(child.into_array(), reference.into()) + .vortex_expect("failed to create FoR array") } else { - FoRArray::try_from_data( - FoRData::try_new(primitive_array, reference.into()) - .vortex_expect("failed to create FoR array"), - ) - .vortex_expect("FoRData is always valid") + FoR::try_new(primitive_array, reference.into()).vortex_expect("failed to create FoR array") } } diff --git a/vortex-cuda/benches/zstd_cuda.rs b/vortex-cuda/benches/zstd_cuda.rs index 7fe68380439..d3e49d03c48 100644 --- a/vortex-cuda/benches/zstd_cuda.rs +++ b/vortex-cuda/benches/zstd_cuda.rs @@ -15,7 +15,7 @@ use futures::executor::block_on; use vortex::array::arrays::VarBinViewArray; use vortex::encodings::zstd::Zstd; use vortex::encodings::zstd::ZstdArray; -use vortex::encodings::zstd::ZstdArrayParts; +use vortex::encodings::zstd::ZstdDataParts; use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::error::vortex_err; @@ -138,7 +138,7 @@ fn benchmark_zstd_cuda_decompress(c: &mut Criterion) { let mut total_time = Duration::ZERO; for _ in 0..iters { - let ZstdArrayParts { + let ZstdDataParts { frames, metadata, .. } = zstd_array.clone().into_data().into_parts(); let exec = block_on(zstd_kernel_prepare(frames, &metadata, &mut cuda_ctx)) diff --git a/vortex-cuda/src/arrow/canonical.rs b/vortex-cuda/src/arrow/canonical.rs index b3bb2824b4c..605d2a9e356 100644 --- a/vortex-cuda/src/arrow/canonical.rs +++ b/vortex-cuda/src/arrow/canonical.rs @@ -7,10 +7,10 @@ use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::ToCanonical; use vortex::array::arrays::StructArray; -use vortex::array::arrays::bool::BoolArrayParts; -use vortex::array::arrays::decimal::DecimalArrayParts; -use vortex::array::arrays::primitive::PrimitiveArrayParts; -use vortex::array::arrays::struct_::StructArrayParts; +use vortex::array::arrays::bool::BoolDataParts; +use vortex::array::arrays::decimal::DecimalDataParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; +use vortex::array::arrays::struct_::StructDataParts; use vortex::array::buffer::BufferHandle; use vortex::dtype::DecimalType; use vortex::error::VortexResult; @@ -66,7 +66,7 @@ fn export_canonical( Canonical::Struct(struct_array) => export_struct(struct_array, ctx).await, Canonical::Primitive(primitive) => { let len = primitive.len(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer, validity, .. } = primitive.into_data().into_parts(); @@ -90,7 +90,7 @@ fn export_canonical( } Canonical::Decimal(decimal) => { let len = decimal.len(); - let DecimalArrayParts { + let DecimalDataParts { values, values_type, validity, @@ -118,7 +118,7 @@ fn export_canonical( let values = extension.storage_array().to_primitive(); let len = extension.len(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer, validity, .. } = values.into_data().into_parts(); @@ -128,13 +128,13 @@ fn export_canonical( export_fixed_size(buffer, len, 0, ctx) } Canonical::Bool(bool_array) => { - let BoolArrayParts { + let BoolDataParts { bits, offset, len, validity, .. - } = bool_array.into_parts(); + } = bool_array.into_data().into_parts(); check_validity_empty(&validity)?; @@ -182,9 +182,9 @@ async fn export_struct( ctx: &mut CudaExecutionCtx, ) -> VortexResult<(ArrowArray, SyncEvent)> { let len = array.len(); - let StructArrayParts { + let StructDataParts { validity, fields, .. - } = array.into_parts(); + } = array.into_data().into_parts(); check_validity_empty(&validity)?; diff --git a/vortex-cuda/src/arrow/varbinview.rs b/vortex-cuda/src/arrow/varbinview.rs index 12edc39c9bf..72ea96437dd 100644 --- a/vortex-cuda/src/arrow/varbinview.rs +++ b/vortex-cuda/src/arrow/varbinview.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use cudarc::driver::LaunchConfig; use cudarc::driver::PushKernelArg; use vortex::array::arrays::VarBinViewArray; -use vortex::array::arrays::varbinview::VarBinViewArrayParts; +use vortex::array::arrays::varbinview::VarBinViewDataParts; use vortex::array::buffer::BufferHandle; use vortex::error::VortexExpect; use vortex::error::VortexResult; @@ -33,7 +33,7 @@ pub(crate) async fn copy_varbinview_to_varbin( ctx: &mut CudaExecutionCtx, ) -> VortexResult { let len = array.len(); - let VarBinViewArrayParts { + let VarBinViewDataParts { views, buffers, validity, diff --git a/vortex-cuda/src/canonical.rs b/vortex-cuda/src/canonical.rs index 0cbfbf718cc..da87deb9f89 100644 --- a/vortex-cuda/src/canonical.rs +++ b/vortex-cuda/src/canonical.rs @@ -13,12 +13,12 @@ use vortex::array::arrays::ExtensionArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; use vortex::array::arrays::VarBinViewArray; -use vortex::array::arrays::bool::BoolArrayParts; -use vortex::array::arrays::decimal::DecimalArrayParts; -use vortex::array::arrays::primitive::PrimitiveArrayParts; -use vortex::array::arrays::struct_::StructArrayParts; +use vortex::array::arrays::bool::BoolDataParts; +use vortex::array::arrays::decimal::DecimalDataParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; +use vortex::array::arrays::struct_::StructDataParts; use vortex::array::arrays::varbinview::BinaryView; -use vortex::array::arrays::varbinview::VarBinViewArrayParts; +use vortex::array::arrays::varbinview::VarBinViewDataParts; use vortex::array::buffer::BufferHandle; use vortex::buffer::BitBuffer; use vortex::buffer::Buffer; @@ -40,12 +40,12 @@ impl CanonicalCudaExt for Canonical { Canonical::Struct(struct_array) => { // Children should all be canonical now let len = struct_array.len(); - let StructArrayParts { + let StructDataParts { fields, struct_fields, validity, .. - } = struct_array.into_parts(); + } = struct_array.into_data().into_parts(); let mut host_fields = vec![]; for field in fields.iter() { @@ -63,19 +63,19 @@ impl CanonicalCudaExt for Canonical { Canonical::Bool(bool) => { // NOTE: update to copy to host when adding buffer handle. // Also update other method to copy validity to host. - let BoolArrayParts { + let BoolDataParts { bits, validity, offset, len, .. - } = bool.into_parts(); + } = bool.into_data().into_parts(); let bits = BitBuffer::new_with_offset(bits.try_into_host()?.await?, offset, len); Ok(Canonical::Bool(BoolArray::new(bits, validity))) } Canonical::Primitive(prim) => { - let PrimitiveArrayParts { + let PrimitiveDataParts { ptype, buffer, validity, @@ -88,7 +88,7 @@ impl CanonicalCudaExt for Canonical { ))) } Canonical::Decimal(decimal) => { - let DecimalArrayParts { + let DecimalDataParts { decimal_dtype, values, values_type, @@ -105,7 +105,7 @@ impl CanonicalCudaExt for Canonical { })) } Canonical::VarBinView(varbinview) => { - let VarBinViewArrayParts { + let VarBinViewDataParts { views, buffers, validity, diff --git a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs index a59a69a173d..19e19119eb0 100644 --- a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs +++ b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs @@ -51,7 +51,7 @@ fn is_dyn_dispatch_compatible(array: &ArrayRef) -> bool { return arr.patches().is_none() && arr.dtype().as_ptype() == PType::F32; } if id == BitPacked::ID { - return array.as_::().patches().is_none(); + return array.as_::().patches(array.len()).is_none(); } if id == Dict::ID { let arr = array.as_::(); @@ -412,7 +412,7 @@ impl FusedPlan { fn walk_bitpacked(&mut self, array: ArrayRef) -> VortexResult { let bp = array.as_::(); - if bp.patches().is_some() { + if bp.patches(array.len()).is_some() { vortex_bail!("Dynamic dispatch does not support BitPackedArray with patches"); } diff --git a/vortex-cuda/src/executor.rs b/vortex-cuda/src/executor.rs index 9ae518540bd..4a14be98968 100644 --- a/vortex-cuda/src/executor.rs +++ b/vortex-cuda/src/executor.rs @@ -21,7 +21,7 @@ use vortex::array::ExecutionCtx; use vortex::array::IntoArray; use vortex::array::arrays::Struct; use vortex::array::arrays::StructArray; -use vortex::array::arrays::struct_::StructArrayParts; +use vortex::array::arrays::struct_::StructDataParts; use vortex::array::buffer::BufferHandle; use vortex::dtype::PType; use vortex::error::VortexResult; @@ -360,12 +360,12 @@ impl CudaArrayExt for ArrayRef { async fn execute_cuda(self, ctx: &mut CudaExecutionCtx) -> VortexResult { if self.encoding_id() == Struct::ID { let len = self.len(); - let StructArrayParts { + let StructDataParts { fields, struct_fields, validity, .. - } = self.try_into::().unwrap().into_parts(); + } = self.try_into::().unwrap().into_data().into_parts(); let mut cuda_fields = Vec::with_capacity(fields.len()); for field in fields.iter() { diff --git a/vortex-cuda/src/hybrid_dispatch/mod.rs b/vortex-cuda/src/hybrid_dispatch/mod.rs index 06ac2c2738d..3832b9a48ad 100644 --- a/vortex-cuda/src/hybrid_dispatch/mod.rs +++ b/vortex-cuda/src/hybrid_dispatch/mod.rs @@ -235,7 +235,6 @@ mod tests { use vortex::array::session::ArraySessionExt; use vortex::encodings::fastlanes; use vortex::encodings::zstd::ZstdBuffers; - use vortex::encodings::zstd::ZstdBuffersData; let mut session = VortexSession::empty(); fastlanes::initialize(&mut session); @@ -256,7 +255,7 @@ mod tests { 0u32.into(), ) .vortex_expect("for"); - let vals = ZstdBuffersData::compress(&vals.into_array(), 3).vortex_expect("zstd"); + let vals = ZstdBuffers::compress(&vals.into_array(), 3).vortex_expect("zstd"); // codes = FoR(BitPacked) let codes = PrimitiveArray::new( diff --git a/vortex-cuda/src/kernel/arrays/dict.rs b/vortex-cuda/src/kernel/arrays/dict.rs index 4015bbcddbb..a8437e0b85b 100644 --- a/vortex-cuda/src/kernel/arrays/dict.rs +++ b/vortex-cuda/src/kernel/arrays/dict.rs @@ -15,10 +15,10 @@ use vortex::array::arrays::Dict; use vortex::array::arrays::DictArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::VarBinViewArray; -use vortex::array::arrays::decimal::DecimalArrayParts; -use vortex::array::arrays::dict::DictArrayParts; -use vortex::array::arrays::primitive::PrimitiveArrayParts; -use vortex::array::arrays::varbinview::VarBinViewArrayParts; +use vortex::array::arrays::decimal::DecimalDataParts; +use vortex::array::arrays::dict::DictDataParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; +use vortex::array::arrays::varbinview::VarBinViewDataParts; use vortex::array::buffer::BufferHandle; use vortex::array::match_each_decimal_value_type; use vortex::array::match_each_integer_ptype; @@ -65,7 +65,7 @@ impl CudaExecute for DictExecutor { #[expect(clippy::cognitive_complexity)] async fn execute_dict_prim(dict: DictArray, ctx: &mut CudaExecutionCtx) -> VortexResult { - let DictArrayParts { values, codes, .. } = dict.into_data().into_parts(); + let DictDataParts { values, codes, .. } = dict.into_data().into_parts(); // Execute both children to get them as primitives on the device let values_canonical = values.execute_cuda(ctx).await?; @@ -93,14 +93,14 @@ async fn execute_dict_prim_typed VortexResult { - let DictArrayParts { - values, - codes, - dtype, - .. - } = dict.into_data().into_parts(); + let dtype = dict.dtype().clone(); + let DictDataParts { values, codes, .. } = dict.into_data().into_parts(); // Execute codes to get them as primitives on the device let codes_prim = codes.execute_cuda(ctx).await?.into_primitive(); @@ -179,14 +175,14 @@ async fn execute_dict_decimal_typed< let codes_len = codes.len(); let codes_len_u64 = codes_len as u64; - let DecimalArrayParts { + let DecimalDataParts { values: values_buffer, validity: values_validity, .. } = values.into_data().into_parts(); let output_validity = values_validity.take(&codes.clone().into_array())?; - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer: codes_buffer, .. } = codes.into_data().into_parts(); @@ -234,19 +230,15 @@ async fn execute_dict_varbinview( dict: DictArray, ctx: &mut CudaExecutionCtx, ) -> VortexResult { - let DictArrayParts { - values, - codes, - dtype, - .. - } = dict.into_data().into_parts(); + let dtype = dict.dtype().clone(); + let DictDataParts { values, codes, .. } = dict.into_data().into_parts(); let codes_prim = codes.execute_cuda(ctx).await?.into_primitive(); let codes_ptype = codes_prim.ptype(); let codes_len = codes_prim.len(); let values_vbv = values.execute_cuda(ctx).await?.into_varbinview(); - let VarBinViewArrayParts { + let VarBinViewDataParts { views: values_views_handle, buffers: values_data_buffers, validity: values_validity, @@ -254,7 +246,7 @@ async fn execute_dict_varbinview( } = values_vbv.into_data().into_parts(); let output_validity = values_validity.take(&codes_prim.clone().into_array())?; - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer: codes_buffer, .. } = codes_prim.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/encodings/alp.rs b/vortex-cuda/src/kernel/encodings/alp.rs index e33a35bbf56..1635927a42e 100644 --- a/vortex-cuda/src/kernel/encodings/alp.rs +++ b/vortex-cuda/src/kernel/encodings/alp.rs @@ -11,7 +11,7 @@ use tracing::instrument; use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::arrays::PrimitiveArray; -use vortex::array::arrays::primitive::PrimitiveArrayParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::array::buffer::BufferHandle; use vortex::array::match_each_unsigned_integer_ptype; use vortex::dtype::NativePType; @@ -46,7 +46,9 @@ impl CudaExecute for ALPExecutor { .try_into::() .map_err(|_| vortex_err!("Expected ALPArray"))?; - match_each_alp_float_ptype!(array.ptype(), |A| { decode_alp::(array, ctx).await }) + match_each_alp_float_ptype!(array.dtype().as_ptype(), |A| { + decode_alp::(array, ctx).await + }) } } @@ -66,7 +68,7 @@ where // Execute child and copy to device let canonical = array.encoded().clone().execute_cuda(ctx).await?; let primitive = canonical.into_primitive(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer, validity, .. } = primitive.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/encodings/bitpacked.rs b/vortex-cuda/src/kernel/encodings/bitpacked.rs index eb7273f1ddf..3bdff6b439d 100644 --- a/vortex-cuda/src/kernel/encodings/bitpacked.rs +++ b/vortex-cuda/src/kernel/encodings/bitpacked.rs @@ -18,7 +18,7 @@ use vortex::array::match_each_integer_ptype; use vortex::dtype::NativePType; use vortex::encodings::fastlanes::BitPacked; use vortex::encodings::fastlanes::BitPackedArray; -use vortex::encodings::fastlanes::BitPackedArrayParts; +use vortex::encodings::fastlanes::BitPackedDataParts; use vortex::encodings::fastlanes::unpack_iter::BitPacked as BitPackedUnpack; use vortex::error::VortexResult; use vortex::error::vortex_ensure; @@ -52,7 +52,7 @@ impl CudaExecute for BitPackedExecutor { let array = Self::try_specialize(array).ok_or_else(|| vortex_err!("Expected BitPackedArray"))?; - match_each_integer_ptype!(array.ptype(), |A| { + match_each_integer_ptype!(array.ptype(array.dtype()), |A| { decode_bitpacked::(array, A::default(), ctx).await }) } @@ -96,14 +96,14 @@ where A: BitPackedUnpack + NativePType + DeviceRepr + Send + Sync + 'static, A::Physical: DeviceRepr + Send + Sync + 'static, { - let BitPackedArrayParts { + let BitPackedDataParts { offset, bit_width, len, packed, patches, validity, - } = array.into_data().into_parts(); + } = BitPacked::into_parts(array); vortex_ensure!(len > 0, "Non empty array"); let offset = offset as usize; @@ -199,7 +199,7 @@ mod tests { // Last two items should be patched let bp_with_patches = BitPacked::encode(&array.into_array(), bw)?; - assert!(bp_with_patches.patches().is_some()); + assert!(bp_with_patches.patches(bp_with_patches.len()).is_some()); let cpu_result = bp_with_patches.to_canonical()?.into_array(); @@ -230,7 +230,7 @@ mod tests { // Last two items should be patched let bp_with_patches = BitPacked::encode(&array.into_array(), 9)?; - assert!(bp_with_patches.patches().is_some()); + assert!(bp_with_patches.patches(bp_with_patches.len()).is_some()); let cpu_result = bp_with_patches.to_canonical()?.into_array(); diff --git a/vortex-cuda/src/kernel/encodings/date_time_parts.rs b/vortex-cuda/src/kernel/encodings/date_time_parts.rs index da21f267d32..a0c4a446567 100644 --- a/vortex-cuda/src/kernel/encodings/date_time_parts.rs +++ b/vortex-cuda/src/kernel/encodings/date_time_parts.rs @@ -13,7 +13,7 @@ use vortex::array::IntoArray; use vortex::array::arrays::ConstantArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::TemporalArray; -use vortex::array::arrays::primitive::PrimitiveArrayParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::array::buffer::BufferHandle; use vortex::array::match_each_signed_integer_ptype; use vortex::array::validity::Validity; @@ -147,15 +147,15 @@ where { let output_len = days.len(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer: days_buffer, .. } = days.into_data().into_parts(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer: seconds_buffer, .. } = seconds.into_data().into_parts(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer: subseconds_buffer, .. } = subseconds.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs b/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs index 134e5c99374..b5aa921a9b6 100644 --- a/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs +++ b/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs @@ -8,9 +8,10 @@ use tracing::instrument; use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::arrays::DecimalArray; -use vortex::array::arrays::primitive::PrimitiveArrayParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::encodings::decimal_byte_parts::DecimalByteParts; -use vortex::encodings::decimal_byte_parts::DecimalBytePartsArrayParts; +use vortex::encodings::decimal_byte_parts::DecimalBytePartsDataParts; +use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::error::vortex_bail; @@ -34,9 +35,12 @@ impl CudaExecute for DecimalBytePartsExecutor { vortex_bail!("cannot downcast to DecimalBytePartsArray") }; - let decimal_dtype = *array.decimal_dtype(); - let DecimalBytePartsArrayParts { msp, .. } = array.into_data().into_parts(); - let PrimitiveArrayParts { + let decimal_dtype = *array + .dtype() + .as_decimal_opt() + .vortex_expect("DecimalBytePartsArray dtype must be decimal"); + let DecimalBytePartsDataParts { msp, .. } = array.into_data().into_parts(); + let PrimitiveDataParts { buffer, ptype, validity, diff --git a/vortex-cuda/src/kernel/encodings/for_.rs b/vortex-cuda/src/kernel/encodings/for_.rs index 57361905c90..7fe9d89d64c 100644 --- a/vortex-cuda/src/kernel/encodings/for_.rs +++ b/vortex-cuda/src/kernel/encodings/for_.rs @@ -12,7 +12,7 @@ use vortex::array::Canonical; use vortex::array::IntoArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::Slice; -use vortex::array::arrays::primitive::PrimitiveArrayParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::array::match_each_integer_ptype; use vortex::array::match_each_native_simd_ptype; use vortex::dtype::NativePType; @@ -52,7 +52,7 @@ impl CudaExecute for FoRExecutor { // Fuse FOR + BP => FFOR if let Some(bitpacked) = array.encoded().as_opt::() { - match_each_integer_ptype!(bitpacked.ptype(), |P| { + match_each_integer_ptype!(bitpacked.ptype(bitpacked.dtype()), |P| { let reference: P = array.reference_scalar().try_into()?; return decode_bitpacked(bitpacked.into_owned(), reference, ctx).await; }) @@ -63,7 +63,7 @@ impl CudaExecute for FoRExecutor { && let Some(bitpacked) = slice_array.child().as_opt::() { let slice_range = slice_array.slice_range().clone(); - let unpacked = match_each_integer_ptype!(bitpacked.ptype(), |P| { + let unpacked = match_each_integer_ptype!(bitpacked.ptype(bitpacked.dtype()), |P| { let reference: P = array.reference_scalar().try_into()?; decode_bitpacked(bitpacked.into_owned(), reference, ctx).await? }); @@ -96,7 +96,7 @@ where // Execute child and copy to device let canonical = array.encoded().clone().execute_cuda(ctx).await?; let primitive = canonical.into_primitive(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer, validity, .. } = primitive.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/encodings/runend.rs b/vortex-cuda/src/kernel/encodings/runend.rs index 6f14ec20c9c..88b64393a5e 100644 --- a/vortex-cuda/src/kernel/encodings/runend.rs +++ b/vortex-cuda/src/kernel/encodings/runend.rs @@ -12,7 +12,7 @@ use vortex::array::Canonical; use vortex::array::IntoArray; use vortex::array::arrays::ConstantArray; use vortex::array::arrays::PrimitiveArray; -use vortex::array::arrays::primitive::PrimitiveArrayParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::array::buffer::BufferHandle; use vortex::array::match_each_native_ptype; use vortex::array::match_each_unsigned_integer_ptype; @@ -21,7 +21,7 @@ use vortex::dtype::NativePType; use vortex::dtype::PType; use vortex::encodings::runend::RunEnd; use vortex::encodings::runend::RunEndArray; -use vortex::encodings::runend::RunEndArrayParts; +use vortex::encodings::runend::RunEndDataParts; use vortex::error::VortexResult; use vortex::error::vortex_bail; use vortex::error::vortex_ensure; @@ -61,7 +61,7 @@ impl CudaExecute for RunEndExecutor { let offset = array.offset(); let output_len = array.len(); - let RunEndArrayParts { ends, values } = array.into_data().into_parts(); + let RunEndDataParts { ends, values, .. } = array.into_data().into_parts(); let values_ptype = PType::try_from(values.dtype())?; let ends_ptype = PType::try_from(ends.dtype())?; @@ -105,14 +105,14 @@ async fn decode_runend_typed() .map_err(|_| vortex_err!("SequenceExecutor can only accept SequenceArray"))?; - let SequenceArrayParts { + let len = array.len(); + let nullability = array.dtype().nullability(); + + let SequenceDataParts { base, multiplier, - len, ptype, - nullability, } = array.into_data().into_parts(); match_each_native_ptype!(ptype, |P| { diff --git a/vortex-cuda/src/kernel/encodings/zigzag.rs b/vortex-cuda/src/kernel/encodings/zigzag.rs index f1feac1bbf3..e7cf89b2181 100644 --- a/vortex-cuda/src/kernel/encodings/zigzag.rs +++ b/vortex-cuda/src/kernel/encodings/zigzag.rs @@ -10,7 +10,7 @@ use tracing::instrument; use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::arrays::PrimitiveArray; -use vortex::array::arrays::primitive::PrimitiveArrayParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::array::match_each_unsigned_integer_ptype; use vortex::dtype::NativePType; use vortex::dtype::PType; @@ -70,7 +70,7 @@ where // Execute child and copy to device let canonical = array.encoded().clone().execute_cuda(ctx).await?; let primitive = canonical.into_primitive(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer, validity, .. } = primitive.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/encodings/zstd.rs b/vortex-cuda/src/kernel/encodings/zstd.rs index a521fe7a27f..e5b7da77fec 100644 --- a/vortex-cuda/src/kernel/encodings/zstd.rs +++ b/vortex-cuda/src/kernel/encodings/zstd.rs @@ -26,7 +26,7 @@ use vortex::buffer::ByteBuffer; use vortex::dtype::DType; use vortex::encodings::zstd::Zstd; use vortex::encodings::zstd::ZstdArray; -use vortex::encodings::zstd::ZstdArrayParts; +use vortex::encodings::zstd::ZstdDataParts; use vortex::encodings::zstd::ZstdMetadata; use vortex::error::VortexExpect; use vortex::error::VortexResult; @@ -206,17 +206,17 @@ impl CudaExecute for ZstdExecutor { dtype = %_other, "Only Binary/Utf8 ZSTD arrays supported on GPU, falling back to CPU" ); - zstd.decompress(ctx.execution_ctx())?.to_canonical() + Zstd::decompress(&zstd, ctx.execution_ctx())?.to_canonical() } } } } async fn decode_zstd(array: ZstdArray, ctx: &mut CudaExecutionCtx) -> VortexResult { - let ZstdArrayParts { + let dtype = array.dtype().clone(); + let ZstdDataParts { frames, metadata, - dtype, validity, n_rows, dictionary, @@ -373,9 +373,7 @@ mod tests { let zstd_array = Zstd::from_var_bin_view(&strings, 3, 0)?; - let cpu_result = zstd_array - .decompress(cuda_ctx.execution_ctx())? - .to_canonical()?; + let cpu_result = Zstd::decompress(&zstd_array, cuda_ctx.execution_ctx())?.to_canonical()?; let gpu_result = ZstdExecutor .execute(zstd_array.into_array(), &mut cuda_ctx) .await?; @@ -410,9 +408,7 @@ mod tests { // 14 strings and 3 values per frame = ceil(14/3) = 5 frames. let zstd_array = Zstd::from_var_bin_view(&strings, 3, 3)?; - let cpu_result = zstd_array - .decompress(cuda_ctx.execution_ctx())? - .to_canonical()?; + let cpu_result = Zstd::decompress(&zstd_array, cuda_ctx.execution_ctx())?.to_canonical()?; let gpu_result = ZstdExecutor .execute(zstd_array.into_array(), &mut cuda_ctx) .await?; diff --git a/vortex-cuda/src/kernel/encodings/zstd_buffers.rs b/vortex-cuda/src/kernel/encodings/zstd_buffers.rs index 1891ee2375c..d44b57615dc 100644 --- a/vortex-cuda/src/kernel/encodings/zstd_buffers.rs +++ b/vortex-cuda/src/kernel/encodings/zstd_buffers.rs @@ -55,7 +55,7 @@ async fn decode_zstd_buffers( let compressed_buffers = plan.compressed_buffers(); if compressed_buffers.is_empty() { - let inner_array = array.build_inner(&[], ctx.session())?; + let inner_array = ZstdBuffers::build_inner(&array, &[], ctx.session())?; return inner_array.execute_cuda(ctx).await; } @@ -158,7 +158,7 @@ async fn decode_zstd_buffers( let output_handle = BufferHandle::new_device(Arc::new(CudaDeviceBuffer::new(device_output))); let decompressed_buffers = plan.split_output_handle(&output_handle)?; - let inner_array = array.build_inner(&decompressed_buffers, ctx.session())?; + let inner_array = ZstdBuffers::build_inner(&array, &decompressed_buffers, ctx.session())?; inner_array.execute_cuda(ctx).await } @@ -223,7 +223,6 @@ mod tests { use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::VarBinViewArray; use vortex::array::assert_arrays_eq; - use vortex::encodings::zstd::ZstdBuffersData; use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::session::VortexSession; @@ -238,7 +237,7 @@ mod tests { .vortex_expect("failed to create execution context"); let input = PrimitiveArray::from_iter(0i64..1024).into_array(); - let compressed = ZstdBuffersData::compress(&input, 3)?; + let compressed = ZstdBuffers::compress(&input, 3)?; let cpu_result = compressed.clone().into_array().to_canonical()?; let gpu_result = ZstdBuffersExecutor @@ -265,7 +264,7 @@ mod tests { "baz", ]) .into_array(); - let compressed = ZstdBuffersData::compress(&input, 3)?; + let compressed = ZstdBuffers::compress(&input, 3)?; let cpu_result = compressed.clone().into_array().to_canonical()?; let gpu_result = ZstdBuffersExecutor diff --git a/vortex-cuda/src/kernel/filter/decimal.rs b/vortex-cuda/src/kernel/filter/decimal.rs index 88e07465968..5e468dadee3 100644 --- a/vortex-cuda/src/kernel/filter/decimal.rs +++ b/vortex-cuda/src/kernel/filter/decimal.rs @@ -4,7 +4,7 @@ use cudarc::driver::DeviceRepr; use vortex::array::Canonical; use vortex::array::arrays::DecimalArray; -use vortex::array::arrays::decimal::DecimalArrayParts; +use vortex::array::arrays::decimal::DecimalDataParts; use vortex::dtype::NativeDecimalType; use vortex::error::VortexResult; use vortex::mask::Mask; @@ -18,7 +18,7 @@ pub(super) async fn filter_decimal VortexResult { - let DecimalArrayParts { + let DecimalDataParts { values, validity, decimal_dtype, diff --git a/vortex-cuda/src/kernel/filter/mod.rs b/vortex-cuda/src/kernel/filter/mod.rs index 588c3e07925..4cf8efbc203 100644 --- a/vortex-cuda/src/kernel/filter/mod.rs +++ b/vortex-cuda/src/kernel/filter/mod.rs @@ -19,7 +19,7 @@ use tracing::instrument; use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::arrays::Filter; -use vortex::array::arrays::filter::FilterArrayParts; +use vortex::array::arrays::filter::FilterDataParts; use vortex::array::buffer::BufferHandle; use vortex::array::match_each_decimal_value_type; use vortex::array::match_each_native_simd_ptype; @@ -53,7 +53,7 @@ impl CudaExecute for FilterExecutor { .try_into::() .map_err(|_| vortex_err!("Expected FilterArray"))?; - let FilterArrayParts { child, mask } = filter_array.into_data().into_parts(); + let FilterDataParts { child, mask } = filter_array.into_data().into_parts(); // Early return for trivial cases. match mask { diff --git a/vortex-cuda/src/kernel/filter/primitive.rs b/vortex-cuda/src/kernel/filter/primitive.rs index 68e39edd60f..f45fe3f2994 100644 --- a/vortex-cuda/src/kernel/filter/primitive.rs +++ b/vortex-cuda/src/kernel/filter/primitive.rs @@ -4,7 +4,7 @@ use cudarc::driver::DeviceRepr; use vortex::array::Canonical; use vortex::array::arrays::PrimitiveArray; -use vortex::array::arrays::primitive::PrimitiveArrayParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::dtype::NativePType; use vortex::error::VortexResult; use vortex::mask::Mask; @@ -22,7 +22,7 @@ pub(super) async fn filter_primitive( where T: NativePType + DeviceRepr + CubFilterable + Send + Sync + 'static, { - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer, validity, .. } = array.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/filter/varbinview.rs b/vortex-cuda/src/kernel/filter/varbinview.rs index 143e3f26ce7..8d673180d58 100644 --- a/vortex-cuda/src/kernel/filter/varbinview.rs +++ b/vortex-cuda/src/kernel/filter/varbinview.rs @@ -3,7 +3,7 @@ use vortex::array::Canonical; use vortex::array::arrays::VarBinViewArray; -use vortex::array::arrays::varbinview::VarBinViewArrayParts; +use vortex::array::arrays::varbinview::VarBinViewDataParts; use vortex::error::VortexResult; use vortex::mask::Mask; @@ -15,7 +15,7 @@ pub(super) async fn filter_varbinview( mask: Mask, ctx: &mut CudaExecutionCtx, ) -> VortexResult { - let VarBinViewArrayParts { + let VarBinViewDataParts { views, buffers, validity, diff --git a/vortex-cuda/src/kernel/patches/mod.rs b/vortex-cuda/src/kernel/patches/mod.rs index 9da4ddf8f51..5d833c1c417 100644 --- a/vortex-cuda/src/kernel/patches/mod.rs +++ b/vortex-cuda/src/kernel/patches/mod.rs @@ -12,7 +12,7 @@ pub mod gpu { use cudarc::driver::DeviceRepr; use cudarc::driver::PushKernelArg; use tracing::instrument; -use vortex::array::arrays::primitive::PrimitiveArrayParts; +use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::array::patches::Patches; use vortex::array::validity::Validity; use vortex::dtype::NativePType; @@ -67,12 +67,12 @@ pub(crate) async fn execute_patches< let patches_len = indices.len(); let patches_len_u64 = patches_len as u64; - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer: indices_buffer, .. } = indices.into_data().into_parts(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer: values_buffer, .. } = values.into_data().into_parts(); @@ -105,7 +105,7 @@ mod tests { use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; - use vortex::array::arrays::primitive::PrimitiveArrayParts; + use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::array::assert_arrays_eq; use vortex::array::buffer::BufferHandle; use vortex::array::builtins::ArrayBuiltins; @@ -167,7 +167,7 @@ mod tests { ) .unwrap(); - let PrimitiveArrayParts { + let PrimitiveDataParts { buffer: cuda_buffer, .. } = values.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/slice/mod.rs b/vortex-cuda/src/kernel/slice/mod.rs index b418e0675f7..5326780b5f9 100644 --- a/vortex-cuda/src/kernel/slice/mod.rs +++ b/vortex-cuda/src/kernel/slice/mod.rs @@ -7,7 +7,7 @@ use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::IntoArray; use vortex::array::arrays::Slice; -use vortex::array::arrays::slice::SliceArrayParts; +use vortex::array::arrays::slice::SliceDataParts; use vortex::error::VortexResult; use vortex::error::vortex_err; @@ -33,7 +33,7 @@ impl CudaExecute for SliceExecutor { ) })?; - let SliceArrayParts { child, range } = slice_array.into_data().into_parts(); + let SliceDataParts { child, range } = slice_array.into_data().into_parts(); let child = child.execute_cuda(ctx).await?; match child { diff --git a/vortex-cuda/src/layout.rs b/vortex-cuda/src/layout.rs index bd9fd51e276..0a8efcaac5f 100644 --- a/vortex-cuda/src/layout.rs +++ b/vortex-cuda/src/layout.rs @@ -26,8 +26,8 @@ use vortex::array::expr::stats::Stat; use vortex::array::expr::stats::StatsProvider; use vortex::array::normalize::NormalizeOptions; use vortex::array::normalize::Operation; -use vortex::array::serde::ArrayParts; use vortex::array::serde::SerializeOptions; +use vortex::array::serde::SerializedArray; use vortex::array::session::ArrayRegistry; use vortex::array::stats::StatsSetRef; use vortex::buffer::BufferString; @@ -249,7 +249,7 @@ impl CudaFlatReader { async move { let segment = segment_fut.await?; - let parts = ArrayParts::from_flatbuffer_and_segment_with_overrides( + let parts = SerializedArray::from_flatbuffer_and_segment_with_overrides( array_tree, segment, &host_buffers, diff --git a/vortex-duckdb/src/exporter/decimal.rs b/vortex-duckdb/src/exporter/decimal.rs index 00026eb9f90..765e76272ca 100644 --- a/vortex-duckdb/src/exporter/decimal.rs +++ b/vortex-duckdb/src/exporter/decimal.rs @@ -6,7 +6,7 @@ use std::marker::PhantomData; use num_traits::ToPrimitive; use vortex::array::ExecutionCtx; use vortex::array::arrays::DecimalArray; -use vortex::array::arrays::decimal::DecimalArrayParts; +use vortex::array::arrays::decimal::DecimalDataParts; use vortex::array::match_each_decimal_value_type; use vortex::buffer::Buffer; use vortex::dtype::BigCast; @@ -42,7 +42,7 @@ pub(crate) fn new_exporter( ctx: &mut ExecutionCtx, ) -> VortexResult> { let len = array.len(); - let DecimalArrayParts { + let DecimalDataParts { validity, decimal_dtype, values_type, diff --git a/vortex-duckdb/src/exporter/list.rs b/vortex-duckdb/src/exporter/list.rs index fab62dcc8cb..f2945123a13 100644 --- a/vortex-duckdb/src/exporter/list.rs +++ b/vortex-duckdb/src/exporter/list.rs @@ -8,7 +8,7 @@ use parking_lot::Mutex; use vortex::array::ExecutionCtx; use vortex::array::arrays::ListArray; use vortex::array::arrays::PrimitiveArray; -use vortex::array::arrays::list::ListArrayParts; +use vortex::array::arrays::list::ListDataParts; use vortex::array::match_each_integer_ptype; use vortex::dtype::IntegerPType; use vortex::error::VortexResult; @@ -45,7 +45,7 @@ pub(crate) fn new_exporter( ) -> VortexResult> { let array_len = array.len(); // Cache an `elements` vector up front so that future exports can reference it. - let ListArrayParts { + let ListDataParts { elements, offsets, validity, diff --git a/vortex-duckdb/src/exporter/list_view.rs b/vortex-duckdb/src/exporter/list_view.rs index 4b76f534ff2..226f399c92f 100644 --- a/vortex-duckdb/src/exporter/list_view.rs +++ b/vortex-duckdb/src/exporter/list_view.rs @@ -8,7 +8,7 @@ use parking_lot::Mutex; use vortex::array::ExecutionCtx; use vortex::array::arrays::ListViewArray; use vortex::array::arrays::PrimitiveArray; -use vortex::array::arrays::listview::ListViewArrayParts; +use vortex::array::arrays::listview::ListViewDataParts; use vortex::array::match_each_integer_ptype; use vortex::dtype::DType; use vortex::dtype::IntegerPType; @@ -47,7 +47,7 @@ pub(crate) fn new_exporter( ctx: &mut ExecutionCtx, ) -> VortexResult> { let len = array.len(); - let ListViewArrayParts { + let ListViewDataParts { elements_dtype, elements, offsets, diff --git a/vortex-duckdb/src/exporter/run_end.rs b/vortex-duckdb/src/exporter/run_end.rs index 0d05cf67a30..af21438130d 100644 --- a/vortex-duckdb/src/exporter/run_end.rs +++ b/vortex-duckdb/src/exporter/run_end.rs @@ -11,7 +11,7 @@ use vortex::array::search_sorted::SearchSorted; use vortex::array::search_sorted::SearchSortedSide; use vortex::dtype::IntegerPType; use vortex::encodings::runend::RunEndArray; -use vortex::encodings::runend::RunEndArrayParts; +use vortex::encodings::runend::RunEndDataParts; use vortex::error::VortexExpect; use vortex::error::VortexResult; @@ -38,7 +38,7 @@ pub(crate) fn new_exporter( ctx: &mut ExecutionCtx, ) -> VortexResult> { let offset = array.offset(); - let RunEndArrayParts { ends, values } = array.into_data().into_parts(); + let RunEndDataParts { ends, values, .. } = array.into_data().into_parts(); let ends = ends.execute::(ctx)?; let values_exporter = new_array_exporter(values.clone(), cache, ctx)?; diff --git a/vortex-duckdb/src/exporter/struct_.rs b/vortex-duckdb/src/exporter/struct_.rs index 8c07ada0533..19ac59c69d9 100644 --- a/vortex-duckdb/src/exporter/struct_.rs +++ b/vortex-duckdb/src/exporter/struct_.rs @@ -5,7 +5,7 @@ use vortex::array::ExecutionCtx; use vortex::array::IntoArray; use vortex::array::arrays::BoolArray; use vortex::array::arrays::StructArray; -use vortex::array::arrays::struct_::StructArrayParts; +use vortex::array::arrays::struct_::StructDataParts; use vortex::array::builtins::ArrayBuiltins; use vortex::error::VortexResult; @@ -27,12 +27,12 @@ pub(crate) fn new_exporter( ctx: &mut ExecutionCtx, ) -> VortexResult> { let len = array.len(); - let StructArrayParts { + let StructDataParts { validity, struct_fields, fields, .. - } = array.into_parts(); + } = array.into_data().into_parts(); let validity = validity.to_array(len).execute::(ctx)?; if validity.to_bit_buffer().true_count() == 0 { diff --git a/vortex-duckdb/src/exporter/varbinview.rs b/vortex-duckdb/src/exporter/varbinview.rs index f0667ee6144..23142b5c4cb 100644 --- a/vortex-duckdb/src/exporter/varbinview.rs +++ b/vortex-duckdb/src/exporter/varbinview.rs @@ -8,7 +8,7 @@ use vortex::array::ExecutionCtx; use vortex::array::arrays::VarBinViewArray; use vortex::array::arrays::varbinview::BinaryView; use vortex::array::arrays::varbinview::Inlined; -use vortex::array::arrays::varbinview::VarBinViewArrayParts; +use vortex::array::arrays::varbinview::VarBinViewDataParts; use vortex::buffer::Buffer; use vortex::buffer::ByteBuffer; use vortex::error::VortexResult; @@ -32,7 +32,7 @@ pub(crate) fn new_exporter( ctx: &mut ExecutionCtx, ) -> VortexResult> { let len = array.len(); - let VarBinViewArrayParts { + let VarBinViewDataParts { validity, dtype, views, diff --git a/vortex-ipc/public-api.lock b/vortex-ipc/public-api.lock index 101ac7940ef..2c79b3ef0cb 100644 --- a/vortex-ipc/public-api.lock +++ b/vortex-ipc/public-api.lock @@ -46,7 +46,7 @@ pub mod vortex_ipc::messages pub enum vortex_ipc::messages::DecoderMessage -pub vortex_ipc::messages::DecoderMessage::Array((vortex_array::serde::ArrayParts, vortex_session::registry::ReadContext, usize)) +pub vortex_ipc::messages::DecoderMessage::Array((vortex_array::serde::SerializedArray, vortex_session::registry::ReadContext, usize)) pub vortex_ipc::messages::DecoderMessage::Buffer(vortex_buffer::ByteBuffer) diff --git a/vortex-ipc/src/messages/decoder.rs b/vortex-ipc/src/messages/decoder.rs index dabbc9e47a8..d0108830aa3 100644 --- a/vortex-ipc/src/messages/decoder.rs +++ b/vortex-ipc/src/messages/decoder.rs @@ -8,7 +8,7 @@ use bytes::Buf; use flatbuffers::root; use flatbuffers::root_unchecked; use vortex_array::ArrayId; -use vortex_array::serde::ArrayParts; +use vortex_array::serde::SerializedArray; use vortex_buffer::AlignedBuf; use vortex_buffer::Alignment; use vortex_buffer::ByteBuffer; @@ -25,7 +25,7 @@ use vortex_session::registry::ReadContext; /// A message decoded from an IPC stream. #[derive(Debug)] pub enum DecoderMessage { - Array((ArrayParts, ReadContext, usize)), + Array((SerializedArray, ReadContext, usize)), Buffer(ByteBuffer), DType(FlatBuffer), } @@ -115,7 +115,7 @@ impl MessageDecoder { MessageHeader::ArrayMessage => { // We don't care about alignment here since ArrayParts will handle it. let body = bytes.copy_to_aligned(body_length, Alignment::new(1)); - let parts = ArrayParts::try_from(body)?; + let parts = SerializedArray::try_from(body)?; let header = msg .header_as_array_message() diff --git a/vortex-layout/src/display.rs b/vortex-layout/src/display.rs index c3c36a06de1..30e029ec8a3 100644 --- a/vortex-layout/src/display.rs +++ b/vortex-layout/src/display.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use futures::future::try_join_all; use termtree::Tree; -use vortex_array::serde::ArrayParts; +use vortex_array::serde::SerializedArray; use vortex_error::VortexResult; use vortex_utils::aliases::hash_map::HashMap; @@ -35,7 +35,7 @@ pub(super) async fn display_tree_with_segment_sizes( let segment_source = segment_source.clone(); async move { let buffer = segment_source.request(segment_id).await?; - let parts = ArrayParts::try_from(buffer)?; + let parts = SerializedArray::try_from(buffer)?; VortexResult::Ok((segment_id, parts.buffer_lengths())) } }); @@ -80,7 +80,7 @@ fn format_flat_layout_buffers( // First, try to get buffer info from inline array_tree if let Some(array_tree) = flat_layout.array_tree() - && let Ok(parts) = ArrayParts::from_array_tree(array_tree.as_ref().to_vec()) + && let Ok(parts) = SerializedArray::from_array_tree(array_tree.as_ref().to_vec()) { return format_buffer_sizes(&parts.buffer_lengths(), *segment_id); } @@ -225,7 +225,7 @@ mod tests { use vortex_array::dtype::Nullability::NonNullable; use vortex_array::dtype::PType; use vortex_array::dtype::StructFields; - use vortex_array::serde::ArrayParts; + use vortex_array::serde::SerializedArray; use vortex_array::validity::Validity; use vortex_buffer::BitBufferMut; use vortex_buffer::buffer; @@ -425,7 +425,7 @@ vortex.chunked, dtype: i32, children: 2, rows: 10 .array_tree() .expect("array_tree should be populated when FLAT_LAYOUT_INLINE_ARRAY_NODE is set"); - let parts = ArrayParts::from_array_tree(array_tree.as_ref().to_vec()) + let parts = SerializedArray::from_array_tree(array_tree.as_ref().to_vec()) .expect("should parse array_tree"); assert_eq!(parts.buffer_lengths(), vec![20]); // 5 i32 values = 20 bytes diff --git a/vortex-layout/src/layouts/flat/reader.rs b/vortex-layout/src/layouts/flat/reader.rs index 779cf12b3eb..7922e1c1560 100644 --- a/vortex-layout/src/layouts/flat/reader.rs +++ b/vortex-layout/src/layouts/flat/reader.rs @@ -14,7 +14,7 @@ use vortex_array::VortexSessionExecute; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; use vortex_array::expr::Expression; -use vortex_array::serde::ArrayParts; +use vortex_array::serde::SerializedArray; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_mask::Mask; @@ -72,10 +72,10 @@ impl FlatReader { let segment = segment_fut.await?; let parts = if let Some(array_tree) = array_tree { // Use the pre-stored flatbuffer from layout metadata combined with segment buffers. - ArrayParts::from_flatbuffer_and_segment(array_tree, segment)? + SerializedArray::from_flatbuffer_and_segment(array_tree, segment)? } else { // Parse the flatbuffer from the segment itself. - ArrayParts::try_from(segment)? + SerializedArray::try_from(segment)? }; parts .decode(&dtype, row_count, &ctx, &session) diff --git a/vortex-python/python/vortex/__init__.py b/vortex-python/python/vortex/__init__.py index 40c3e7c4a91..5396f50b7a0 100644 --- a/vortex-python/python/vortex/__init__.py +++ b/vortex-python/python/vortex/__init__.py @@ -74,7 +74,7 @@ Utf8Scalar, scalar, ) -from ._lib.serde import ArrayContext, ArrayParts # pyright: ignore[reportMissingModuleSource] +from ._lib.serde import ArrayContext, SerializedArray # pyright: ignore[reportMissingModuleSource] from .arrays import ( Array, PyArray, @@ -177,7 +177,7 @@ "ExtensionScalar", # Serde "ArrayContext", - "ArrayParts", + "SerializedArray", # Pickle "_unpickle_array", # File diff --git a/vortex-python/python/vortex/_lib/serde.pyi b/vortex-python/python/vortex/_lib/serde.pyi index 626946a7e6c..b4d052b3e47 100644 --- a/vortex-python/python/vortex/_lib/serde.pyi +++ b/vortex-python/python/vortex/_lib/serde.pyi @@ -10,9 +10,9 @@ from .arrays import Array from .dtype import DType @final -class ArrayParts: +class SerializedArray: @staticmethod - def parse(data: bytes) -> ArrayParts: ... + def parse(data: bytes) -> SerializedArray: ... @property def metadata(self) -> bytes | None: ... @property @@ -22,7 +22,7 @@ class ArrayParts: @property def nchildren(self) -> int: ... @property - def children(self) -> list[ArrayParts]: ... + def children(self) -> list[SerializedArray]: ... def decode(self, ctx: ArrayContext, dtype: DType, len: int) -> pa.Array[pa.Scalar[pa.DataType]]: ... @final diff --git a/vortex-python/python/vortex/arrays.py b/vortex-python/python/vortex/arrays.py index 4180510d501..5181400cb32 100644 --- a/vortex-python/python/vortex/arrays.py +++ b/vortex-python/python/vortex/arrays.py @@ -13,7 +13,7 @@ from vortex._lib.dtype import DType # pyright: ignore[reportMissingModuleSource] from vortex._lib.serde import ( # pyright: ignore[reportMissingModuleSource] ArrayContext, - ArrayParts, + SerializedArray, decode_ipc_array_buffers, ) @@ -463,12 +463,12 @@ def dtype(self) -> DType: @classmethod @abc.abstractmethod - def decode(cls, parts: ArrayParts, ctx: ArrayContext, dtype: DType, len: int) -> Array: + def decode(cls, parts: SerializedArray, ctx: ArrayContext, dtype: DType, len: int) -> Array: """Decode an array from its component parts. - :class:`ArrayParts` contains the metadata, buffers and child :class:`ArrayParts` that represent the - current array. Implementations of this function should validate this information, and then construct - a new array. + :class:`SerializedArray` contains the metadata, buffers and child :class:`SerializedArray` + that represent the current array. Implementations of this function should validate this + information, and then construct a new array. """ diff --git a/vortex-python/python/vortex/serde.py b/vortex-python/python/vortex/serde.py index cb1f7f6c9a0..c30ceef4058 100644 --- a/vortex-python/python/vortex/serde.py +++ b/vortex-python/python/vortex/serde.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -from vortex._lib.serde import ArrayContext, ArrayParts # pyright: ignore[reportMissingModuleSource] +from vortex._lib.serde import ArrayContext, SerializedArray # pyright: ignore[reportMissingModuleSource] -__all__ = ["ArrayParts", "ArrayContext"] +__all__ = ["SerializedArray", "ArrayContext"] diff --git a/vortex-python/src/arrays/py/array.rs b/vortex-python/src/arrays/py/array.rs index 9163ae0c37d..1653ed4b2aa 100644 --- a/vortex-python/src/arrays/py/array.rs +++ b/vortex-python/src/arrays/py/array.rs @@ -8,7 +8,12 @@ use pyo3::FromPyObject; use pyo3::Py; use pyo3::PyAny; use pyo3::prelude::*; +use vortex::array::Array; +use vortex::array::ArrayParts; +use vortex::array::ArrayRef; +use vortex::array::IntoArray; use vortex::array::stats::ArrayStats; +use vortex::array::stats::StatsSet; use vortex::dtype::DType; use crate::arrays::py::PyPythonArray; @@ -55,3 +60,18 @@ impl<'py> IntoPyObject<'py> for PythonArray { Ok(self.object.bind(py).to_owned()) } } + +impl IntoArray for PythonArray { + fn into_array(self) -> ArrayRef { + let vtable = self.vtable.clone(); + let dtype = self.dtype.clone(); + let len = self.len; + let stats = StatsSet::from(self.stats.clone()); + match Array::try_from_parts(ArrayParts::new(vtable, dtype, len, self)) { + Ok(array) => array.with_stats_set(stats).into_array(), + Err(err) => unreachable!( + "PythonArray metadata extracted from PyPythonArray must be valid: {err}" + ), + } + } +} diff --git a/vortex-python/src/arrays/py/vtable.rs b/vortex-python/src/arrays/py/vtable.rs index 3f323295020..dbbc767f253 100644 --- a/vortex-python/src/arrays/py/vtable.rs +++ b/vortex-python/src/arrays/py/vtable.rs @@ -4,9 +4,6 @@ use std::hash::Hash; use std::sync::Arc; -use pyo3::intern; -use pyo3::prelude::*; -use pyo3::types::PyBytes; use vortex::array::Array; use vortex::array::ArrayId; use vortex::array::ArrayRef; @@ -15,19 +12,15 @@ use vortex::array::ExecutionCtx; use vortex::array::ExecutionResult; use vortex::array::OperationsVTable; use vortex::array::Precision; -use vortex::array::RawMetadata; -use vortex::array::SerializeMetadata; use vortex::array::VTable; use vortex::array::ValidityVTable; use vortex::array::buffer::BufferHandle; -use vortex::array::serde::ArrayChildren; -use vortex::array::stats::ArrayStats; use vortex::array::validity::Validity; use vortex::array::vtable; use vortex::dtype::DType; use vortex::error::VortexResult; +use vortex::error::vortex_bail; use vortex::error::vortex_ensure; -use vortex::error::vortex_err; use vortex::error::vortex_panic; use vortex::scalar::Scalar; use vortex::session::VortexSession; @@ -45,28 +38,18 @@ pub struct PythonVTable { impl VTable for PythonVTable { type ArrayData = PythonArray; - type Metadata = RawMetadata; type OperationsVTable = Self; type ValidityVTable = Self; - fn vtable(array: &Self::ArrayData) -> &Self { - &array.vtable - } - fn id(&self) -> ArrayId { self.id.clone() } - fn len(array: &PythonArray) -> usize { - array.len - } - - fn dtype(array: &PythonArray) -> &DType { - &array.dtype - } - - fn stats(array: &PythonArray) -> &ArrayStats { - &array.stats + fn validate(&self, data: &PythonArray, dtype: &DType, len: usize) -> VortexResult<()> { + vortex_ensure!(data.vtable.id == self.id, "PythonArray vtable id mismatch"); + vortex_ensure!(&data.dtype == dtype, "PythonArray dtype mismatch"); + vortex_ensure!(data.len == len, "PythonArray len mismatch"); + Ok(()) } fn array_hash(array: &PythonArray, state: &mut H, _precision: Precision) { @@ -101,51 +84,21 @@ impl VTable for PythonVTable { vortex_panic!("PythonArray child_name index {idx} out of bounds") } - fn metadata(array: ArrayView<'_, Self>) -> VortexResult { - Python::attach(|py| { - let obj = array.object.bind(py); - if !obj - .hasattr(intern!(py, "metadata")) - .map_err(|e| vortex_err!("{}", e))? - { - // The class does not have a metadata attribute so does not support serialization. - return Ok(RawMetadata(vec![])); - } - - let bytes = obj - .call_method(intern!(py, "__vx_metadata__"), (), None) - .map_err(|e| vortex_err!("{}", e))? - .cast::() - .map_err(|_| vortex_err!("Expected array metadata to be Python bytes"))? - .as_bytes() - .to_vec(); - - Ok(RawMetadata(bytes)) - }) - } - - fn serialize(metadata: Self::Metadata) -> VortexResult>> { - Ok(Some(metadata.serialize())) + fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { + Ok(None) } fn deserialize( - bytes: &[u8], + &self, _dtype: &DType, _len: usize, + bytes: &[u8], _buffers: &[BufferHandle], + _children: &dyn vortex::array::serde::ArrayChildren, _session: &VortexSession, - ) -> VortexResult { - Ok(RawMetadata(bytes.to_vec())) - } - - fn build( - _dtype: &DType, - _len: usize, - _metadata: &Self::Metadata, - _buffers: &[BufferHandle], - _children: &dyn ArrayChildren, ) -> VortexResult { - todo!() + _ = bytes; + vortex_bail!("PythonArray deserialization is not supported"); } fn slots(_array: ArrayView<'_, Self>) -> &[Option] { diff --git a/vortex-python/src/serde/mod.rs b/vortex-python/src/serde/mod.rs index e3290821194..c4e4cc95948 100644 --- a/vortex-python/src/serde/mod.rs +++ b/vortex-python/src/serde/mod.rs @@ -21,7 +21,7 @@ use crate::error::PyVortexResult; use crate::install_module; use crate::serde::context::PyArrayContext; use crate::serde::context::PyReadContext; -use crate::serde::parts::PyArrayParts; +use crate::serde::parts::PySerializedArray; /// Register serde functions and classes. pub(crate) fn init(py: Python, parent: &Bound) -> PyResult<()> { @@ -29,7 +29,7 @@ pub(crate) fn init(py: Python, parent: &Bound) -> PyResult<()> { parent.add_submodule(&m)?; install_module("vortex._lib.serde", &m)?; - m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_function(wrap_pyfunction!(decode_ipc_array, &m)?)?; diff --git a/vortex-python/src/serde/parts.rs b/vortex-python/src/serde/parts.rs index 2decc0a926a..e05a6b6d1ac 100644 --- a/vortex-python/src/serde/parts.rs +++ b/vortex-python/src/serde/parts.rs @@ -11,7 +11,7 @@ use pyo3::intern; use pyo3::prelude::PyAnyMethods; use pyo3::pyclass; use pyo3::pymethods; -use vortex::array::serde::ArrayParts; +use vortex::array::serde::SerializedArray; use vortex::buffer::ByteBuffer; use crate::SESSION; @@ -20,34 +20,34 @@ use crate::dtype::PyDType; use crate::error::PyVortexResult; use crate::serde::context::PyReadContext; -/// ArrayParts is a parsed representation of a serialized array. +/// SerializedArray is a parsed representation of a serialized array. /// /// It can be decoded into a full array using the `decode` method. -#[pyclass(name = "ArrayParts", module = "vortex", frozen)] -pub(crate) struct PyArrayParts(ArrayParts); +#[pyclass(name = "SerializedArray", module = "vortex", frozen)] +pub(crate) struct PySerializedArray(SerializedArray); -impl Deref for PyArrayParts { - type Target = ArrayParts; +impl Deref for PySerializedArray { + type Target = SerializedArray; fn deref(&self) -> &Self::Target { &self.0 } } -impl From for PyArrayParts { - fn from(parts: ArrayParts) -> Self { +impl From for PySerializedArray { + fn from(parts: SerializedArray) -> Self { Self(parts) } } #[pymethods] -impl PyArrayParts { +impl PySerializedArray { /// Parse a serialized array into its parts. #[staticmethod] - fn parse(data: &[u8]) -> PyVortexResult { + fn parse(data: &[u8]) -> PyVortexResult { // TODO(ngates): create a buffer from a slice of bytes? let buffer = ByteBuffer::copy_from(data); - Ok(PyArrayParts(ArrayParts::try_from(buffer)?)) + Ok(PySerializedArray(SerializedArray::try_from(buffer)?)) } /// Decode the array parts into a full array. @@ -115,12 +115,12 @@ impl PyArrayParts { self.0.nchildren() } - /// Return the child :class:`~vortex.ArrayParts` of the array. + /// Return the child :class:`~vortex.SerializedArray` of the array. #[getter] - fn children(&self) -> Vec { + fn children(&self) -> Vec { (0..self.0.nchildren()) .map(|idx| self.0.child(idx)) - .map(PyArrayParts) + .map(PySerializedArray) .collect() } } diff --git a/vortex-python/test/test_pyarray.py b/vortex-python/test/test_pyarray.py index 5713ff669f0..f29ca41408f 100644 --- a/vortex-python/test/test_pyarray.py +++ b/vortex-python/test/test_pyarray.py @@ -91,7 +91,7 @@ def encode(cls, array: pa.Array[pa.Scalar[pa.DataType]], config: ChunkConfig | N @override @classmethod - def decode(cls, parts: vx.ArrayParts, ctx: vx.ArrayContext, dtype: vx.DType, len: int) -> vx.Array: + def decode(cls, parts: vx.SerializedArray, ctx: vx.ArrayContext, dtype: vx.DType, len: int) -> vx.Array: """Decode the serialized array parts into an array.""" assert pco raise NotImplementedError diff --git a/vortex-tui/src/browse/app.rs b/vortex-tui/src/browse/app.rs index a6292d071a8..340f19fbed6 100644 --- a/vortex-tui/src/browse/app.rs +++ b/vortex-tui/src/browse/app.rs @@ -352,7 +352,7 @@ impl AppState { #[cfg(feature = "native")] pub(crate) async fn load_flat_data(&mut self) { use vortex::array::MaskFuture; - use vortex::array::serde::ArrayParts; + use vortex::array::serde::SerializedArray; use vortex::expr::root; let layout = &self.cursor.layout().clone(); @@ -384,7 +384,7 @@ impl AppState { .await .vortex_expect("Failed to read segment"); self.cached_flatbuffer_size = Some( - ArrayParts::try_from(segment) + SerializedArray::try_from(segment) .vortex_expect("Failed to parse segment") .metadata() .len(), diff --git a/vortex-tui/src/wasm.rs b/vortex-tui/src/wasm.rs index f8acae67c61..8b643161fe2 100644 --- a/vortex-tui/src/wasm.rs +++ b/vortex-tui/src/wasm.rs @@ -14,7 +14,7 @@ use ratzilla::CanvasBackend; use ratzilla::WebRenderer; use ratzilla::ratatui::Terminal; use vortex::array::MaskFuture; -use vortex::array::serde::ArrayParts; +use vortex::array::serde::SerializedArray; use vortex::error::VortexExpect; use vortex::expr::root; use vortex::layout::layouts::flat::Flat; @@ -102,7 +102,7 @@ async fn load_flatbuffer_size( .request(segment_id) .await .vortex_expect("Failed to read segment"); - ArrayParts::try_from(segment) + SerializedArray::try_from(segment) .vortex_expect("Failed to parse segment") .metadata() .len() diff --git a/vortex-web/crate/src/wasm.rs b/vortex-web/crate/src/wasm.rs index bd4782079ff..1139907f47f 100644 --- a/vortex-web/crate/src/wasm.rs +++ b/vortex-web/crate/src/wasm.rs @@ -25,7 +25,7 @@ use vortex::array::VortexSessionExecute; use vortex::array::arrow::ArrowArrayExecutor; use vortex::array::buffer::BufferHandle; use vortex::array::dtype::DType; -use vortex::array::serde::ArrayParts; +use vortex::array::serde::SerializedArray; use vortex::array::stream::ArrayStream; use vortex::buffer::Alignment; use vortex::buffer::ByteBufferMut; @@ -361,7 +361,8 @@ impl VortexFileHandle { .await .map_err(|e| JsValue::from_str(&e.to_string()))?; - let parts = ArrayParts::try_from(buf).map_err(|e| JsValue::from_str(&e.to_string()))?; + let parts = + SerializedArray::try_from(buf).map_err(|e| JsValue::from_str(&e.to_string()))?; let array = parts .decode(&dtype, row_count, ctx, &self.session) @@ -383,8 +384,6 @@ impl VortexFileHandle { array_path: Vec, buffer_index: usize, ) -> Result { - use vortex::array::ArrayVisitor; - let ctx = self .array_read_ctx .as_ref() @@ -410,7 +409,8 @@ impl VortexFileHandle { .await .map_err(|e| JsValue::from_str(&e.to_string()))?; - let parts = ArrayParts::try_from(buf).map_err(|e| JsValue::from_str(&e.to_string()))?; + let parts = + SerializedArray::try_from(buf).map_err(|e| JsValue::from_str(&e.to_string()))?; let root_array = parts .decode(&dtype, row_count, ctx, &self.session) @@ -451,8 +451,6 @@ impl VortexFileHandle { array_path: Vec, row_limit: u32, ) -> Result { - use vortex::array::ArrayVisitor; - let ctx = self .array_read_ctx .as_ref() @@ -478,7 +476,8 @@ impl VortexFileHandle { .await .map_err(|e| JsValue::from_str(&e.to_string()))?; - let parts = ArrayParts::try_from(buf).map_err(|e| JsValue::from_str(&e.to_string()))?; + let parts = + SerializedArray::try_from(buf).map_err(|e| JsValue::from_str(&e.to_string()))?; let root_array = parts .decode(&dtype, row_count, ctx, &self.session) @@ -583,7 +582,7 @@ fn build_layout_tree( let array_encoding_tree = layout.as_opt::().and_then(|flat| { let tree_buf = flat.array_tree()?; let ctx = flat.array_ctx(); - let parts = ArrayParts::from_array_tree(tree_buf.as_ref().to_vec()).ok()?; + let parts = SerializedArray::from_array_tree(tree_buf.as_ref().to_vec()).ok()?; Some(build_array_encoding_tree(&parts, ctx)) }); @@ -612,7 +611,7 @@ fn sum_metadata_bytes(layout: &LayoutRef) -> VortexResult { /// Recursively build the array encoding tree from `ArrayParts` (used for inline trees /// where we don't have a fully decoded array). -fn build_array_encoding_tree(parts: &ArrayParts, ctx: &ReadContext) -> ArrayEncodingNodeJson { +fn build_array_encoding_tree(parts: &SerializedArray, ctx: &ReadContext) -> ArrayEncodingNodeJson { let encoding = ctx .resolve(parts.encoding_id()) .map(|id| id.to_string()) @@ -638,8 +637,6 @@ fn build_array_encoding_tree(parts: &ArrayParts, ctx: &ReadContext) -> ArrayEnco /// Recursively build the array encoding tree from a fully decoded array, /// extracting dtype, child names, and buffer names from the encoding vtables. fn build_array_encoding_tree_from_array(array: &ArrayRef) -> ArrayEncodingNodeJson { - use vortex::array::ArrayVisitor; - let encoding = array.encoding_id().to_string(); let dtype = array.dtype().to_string(); let buffer_names = array.buffer_names(); diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index d2928a81a8f..8a4cfa53f59 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -27,6 +27,7 @@ use vortex::array::session::ArraySession; use vortex::dtype::PType; use vortex::encodings::alp::RDEncoder; use vortex::encodings::alp::alp_encode; +use vortex::encodings::fastlanes::Delta; use vortex::encodings::fastlanes::DeltaData; use vortex::encodings::fastlanes::FoR; use vortex::encodings::fastlanes::delta_compress; @@ -36,6 +37,7 @@ use vortex::encodings::pco::Pco; use vortex::encodings::runend::RunEnd; use vortex::encodings::sequence::sequence_encode; use vortex::encodings::zigzag::zigzag_encode; +use vortex::encodings::zstd::Zstd; use vortex::encodings::zstd::ZstdData; use vortex_sequence::Sequence; use vortex_session::VortexSession; @@ -162,10 +164,9 @@ fn bench_delta_compress_u32(bencher: Bencher) { fn bench_delta_decompress_u32(bencher: Bencher) { let (uint_array, ..) = setup_primitive_arrays(); let (bases, deltas) = delta_compress(&uint_array, &mut SESSION.create_execution_ctx()).unwrap(); - let compressed = - DeltaData::try_new(bases.into_array(), deltas.into_array(), 0, uint_array.len()) - .unwrap() - .into_array(); + let compressed = Delta::try_new(bases.into_array(), deltas.into_array(), 0, uint_array.len()) + .unwrap() + .into_array(); with_byte_counter(bencher, NUM_VALUES * 4) .with_inputs(|| &compressed) @@ -327,9 +328,13 @@ fn bench_zstd_compress_u32(bencher: Bencher) { #[divan::bench(name = "zstd_decompress_u32")] fn bench_zstd_decompress_u32(bencher: Bencher) { let (uint_array, ..) = setup_primitive_arrays(); - let compressed = ZstdData::from_array(uint_array.into_array(), 3, 8192) - .unwrap() - .into_array(); + let dtype = uint_array.dtype().clone(); + let compressed = Zstd::try_new( + dtype, + ZstdData::from_array(uint_array.into_array(), 3, 8192).unwrap(), + ) + .unwrap() + .into_array(); with_byte_counter(bencher, NUM_VALUES * 4) .with_inputs(|| &compressed) @@ -408,9 +413,13 @@ fn bench_zstd_compress_string(bencher: Bencher) { fn bench_zstd_decompress_string(bencher: Bencher) { let varbinview_arr = VarBinViewArray::from_iter_str(gen_varbin_words(NUM_VALUES as usize, 0.00005)); - let compressed = ZstdData::from_array(varbinview_arr.clone().into_array(), 3, 8192) - .unwrap() - .into_array(); + let dtype = varbinview_arr.dtype().clone(); + let compressed = Zstd::try_new( + dtype, + ZstdData::from_array(varbinview_arr.clone().into_array(), 3, 8192).unwrap(), + ) + .unwrap() + .into_array(); let nbytes = varbinview_arr.into_array().nbytes() as u64; with_byte_counter(bencher, nbytes)