diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 1e4a8a58ee5..8ee3c97f17b 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -2302,7 +2302,7 @@ pub fn vortex_array::arrays::Extension::scalar_at(array: vortex_array::ArrayView impl vortex_array::VTable for vortex_array::arrays::Extension -pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData +pub type vortex_array::arrays::Extension::ArrayData = vortex_array::EmptyArrayData pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension @@ -2338,7 +2338,7 @@ pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::validate(&self, _data: &vortex_array::EmptyArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityChild for vortex_array::arrays::Extension @@ -2368,44 +2368,16 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::Ex pub fn vortex_array::arrays::Extension::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::Extension>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::extension::ExtensionData - -impl vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::ext_dtype(&self) -> &vortex_array::dtype::extension::ExtDTypeRef - -pub fn vortex_array::arrays::extension::ExtensionData::new(ext_dtype: vortex_array::dtype::extension::ExtDTypeRef, storage_dtype: &vortex_array::dtype::DType) -> Self - -pub unsafe fn vortex_array::arrays::extension::ExtensionData::new_unchecked(ext_dtype: vortex_array::dtype::extension::ExtDTypeRef, storage_dtype: &vortex_array::dtype::DType) -> Self - -pub fn vortex_array::arrays::extension::ExtensionData::try_new(ext_dtype: vortex_array::dtype::extension::ExtDTypeRef, storage_dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult - -impl core::clone::Clone for vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::clone(&self) -> vortex_array::arrays::extension::ExtensionData - -impl core::fmt::Debug for vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -impl core::fmt::Display for vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -impl vortex_array::ArrayEq for vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool - -impl vortex_array::ArrayHash for vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) - pub trait vortex_array::arrays::extension::ExtensionArrayExt: vortex_array::TypedArrayRef +pub fn vortex_array::arrays::extension::ExtensionArrayExt::ext_dtype(&self) -> &vortex_array::dtype::extension::ExtDTypeRef + pub fn vortex_array::arrays::extension::ExtensionArrayExt::storage_array(&self) -> &vortex_array::ArrayRef impl> vortex_array::arrays::extension::ExtensionArrayExt for T +pub fn T::ext_dtype(&self) -> &vortex_array::dtype::extension::ExtDTypeRef + pub fn T::storage_array(&self) -> &vortex_array::ArrayRef pub type vortex_array::arrays::extension::ExtensionArray = vortex_array::Array @@ -5558,7 +5530,7 @@ pub fn vortex_array::arrays::Extension::scalar_at(array: vortex_array::ArrayView impl vortex_array::VTable for vortex_array::arrays::Extension -pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData +pub type vortex_array::arrays::Extension::ArrayData = vortex_array::EmptyArrayData pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension @@ -5594,7 +5566,7 @@ pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::validate(&self, _data: &vortex_array::EmptyArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityChild for vortex_array::arrays::Extension @@ -19464,7 +19436,7 @@ pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::array impl vortex_array::VTable for vortex_array::arrays::Extension -pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData +pub type vortex_array::arrays::Extension::ArrayData = vortex_array::EmptyArrayData pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension @@ -19500,7 +19472,7 @@ pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::validate(&self, _data: &vortex_array::EmptyArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Filter @@ -20436,7 +20408,7 @@ pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::array impl vortex_array::VTable for vortex_array::arrays::Extension -pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData +pub type vortex_array::arrays::Extension::ArrayData = vortex_array::EmptyArrayData pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension @@ -20472,7 +20444,7 @@ pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::validate(&self, _data: &vortex_array::EmptyArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Filter @@ -22686,10 +22658,6 @@ impl vortex_array::ArrayEq for vortex_array::arrays::dict::DictData pub fn vortex_array::arrays::dict::DictData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool -impl vortex_array::ArrayEq for vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool - impl vortex_array::ArrayEq for vortex_array::arrays::filter::FilterData pub fn vortex_array::arrays::filter::FilterData::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool @@ -22794,10 +22762,6 @@ impl vortex_array::ArrayHash for vortex_array::arrays::dict::DictData pub fn vortex_array::arrays::dict::DictData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) -impl vortex_array::ArrayHash for vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) - impl vortex_array::ArrayHash for vortex_array::arrays::filter::FilterData pub fn vortex_array::arrays::filter::FilterData::array_hash(&self, state: &mut H, precision: vortex_array::Precision) @@ -23132,7 +23096,7 @@ pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::array impl vortex_array::VTable for vortex_array::arrays::Extension -pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData +pub type vortex_array::arrays::Extension::ArrayData = vortex_array::EmptyArrayData pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension @@ -23168,7 +23132,7 @@ pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::validate(&self, _data: &vortex_array::EmptyArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Filter @@ -24352,7 +24316,7 @@ pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::array impl vortex_array::VTable for vortex_array::arrays::Extension -pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData +pub type vortex_array::arrays::Extension::ArrayData = vortex_array::EmptyArrayData pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension @@ -24388,7 +24352,7 @@ pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::validate(&self, _data: &vortex_array::EmptyArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Filter diff --git a/vortex-array/src/arrays/extension/array.rs b/vortex-array/src/arrays/extension/array.rs index 0a79774a2b5..b42f7b63603 100644 --- a/vortex-array/src/arrays/extension/array.rs +++ b/vortex-array/src/arrays/extension/array.rs @@ -1,13 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::fmt::Display; -use std::fmt::Formatter; - use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_error::vortex_ensure_eq; use crate::ArrayRef; +use crate::EmptyArrayData; use crate::array::Array; use crate::array::ArrayParts; use crate::array::TypedArrayRef; @@ -22,113 +21,14 @@ pub(super) const STORAGE_SLOT: usize = 0; pub(super) const NUM_SLOTS: usize = 1; pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["storage"]; -/// An extension array that wraps another array with additional type information. -/// -/// **⚠️ Unstable API**: This is an experimental feature that may change significantly -/// in future versions. The extension type system is still evolving. -/// -/// Unlike Apache Arrow's extension arrays, Vortex extension arrays provide a more flexible -/// mechanism for adding semantic meaning to existing array types without requiring -/// changes to the core type system. -/// -/// ## Design Philosophy -/// -/// Extension arrays serve as a type-safe wrapper that: -/// - Preserves the underlying storage format and operations -/// - Adds semantic type information via `ExtDType` -/// - Enables custom serialization and deserialization logic -/// - Allows domain-specific interpretations of generic data -/// -/// ## Storage and Type Relationship -/// -/// The extension array maintains a strict contract: -/// - **Storage array**: Contains the actual data in a standard Vortex encoding -/// - **Extension type**: Defines how to interpret the storage data semantically -/// - **Type safety**: The storage array's dtype must match the extension type's storage dtype -/// -/// ## Use Cases -/// -/// Extension arrays are ideal for: -/// - **Custom numeric types**: Units of measurement, currencies -/// - **Temporal types**: Custom date/time formats, time zones, calendars -/// - **Domain-specific types**: UUIDs, IP addresses, geographic coordinates -/// - **Encoded types**: Base64 strings, compressed data, encrypted values -/// -/// ## Validity and Operations -/// -/// Extension arrays delegate validity and most operations to their storage array: -/// - Validity is inherited from the underlying storage -/// - Slicing preserves the extension type -/// - Scalar access wraps storage scalars with extension metadata -#[derive(Clone, Debug)] -pub struct ExtensionData { - /// The storage dtype. This **must** be a [`Extension::DType`] variant. - pub(super) ext_dtype: ExtDTypeRef, -} - -impl Display for ExtensionData { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "ext_dtype: {}", self.ext_dtype) - } -} - -impl ExtensionData { - /// Constructs a new `ExtensionArray`. - /// - /// # Panics - /// - /// Panics if the storage array in not compatible with the extension dtype. - pub fn new(ext_dtype: ExtDTypeRef, storage_dtype: &DType) -> Self { - Self::try_new(ext_dtype, storage_dtype).vortex_expect("Failed to create `ExtensionArray`") - } - - /// Tries to construct a new `ExtensionArray`. - /// - /// # Errors - /// - /// Returns an error if the storage array in not compatible with the extension dtype. - pub fn try_new(ext_dtype: ExtDTypeRef, storage_dtype: &DType) -> VortexResult { - // TODO(connor): Replace these statements once we add `validate_storage_array`. - // ext_dtype.validate_storage_array(&storage_array)?; - assert_eq!( - ext_dtype.storage_dtype(), - storage_dtype, - "ExtensionArray: storage_dtype must match storage array DType", - ); - - // SAFETY: we validate that the inputs are valid above. - Ok(unsafe { Self::new_unchecked(ext_dtype, storage_dtype) }) - } - - /// Creates a new `ExtensionArray`. - /// - /// # Safety - /// - /// The caller must ensure that the storage array is compatible with the extension dtype. In - /// other words, they must know that `ext_dtype.validate_storage_array(&storage_array)` has been - /// called successfully on this storage array. - pub unsafe fn new_unchecked(ext_dtype: ExtDTypeRef, storage_dtype: &DType) -> Self { - // TODO(connor): Replace these statements once we add `validate_storage_array`. - // #[cfg(debug_assertions)] - // ext_dtype - // .validate_storage_array(&storage_array) - // .vortex_expect("[Debug Assertion]: Invalid storage array for `ExtensionArray`"); - debug_assert_eq!( - ext_dtype.storage_dtype(), - storage_dtype, - "ExtensionArray: storage_dtype must match storage array DType", - ); - - Self { ext_dtype } - } - - /// The extension dtype of this array. - pub fn ext_dtype(&self) -> &ExtDTypeRef { - &self.ext_dtype +pub trait ExtensionArrayExt: TypedArrayRef { + fn ext_dtype(&self) -> &ExtDTypeRef { + self.as_ref() + .dtype() + .as_extension_opt() + .vortex_expect("extension array somehow did not have an extension dtype") } -} -pub trait ExtensionArrayExt: TypedArrayRef { fn storage_array(&self) -> &ArrayRef { self.as_ref().slots()[STORAGE_SLOT] .as_ref() @@ -144,26 +44,24 @@ impl Array { /// /// Panics if the storage array is not compatible with the extension dtype. pub fn new(ext_dtype: ExtDTypeRef, storage_array: ArrayRef) -> Self { - let dtype = DType::Extension(ext_dtype.clone()); - let len = storage_array.len(); - let data = ExtensionData::new(ext_dtype, storage_array.dtype()); - unsafe { - Array::from_parts_unchecked( - ArrayParts::new(Extension, dtype, len, data).with_slots(vec![Some(storage_array)]), - ) - } + Self::try_new(ext_dtype, storage_array).vortex_expect("Unable to create `ExtensionArray`") } /// Tries to construct a new `ExtensionArray`. pub fn try_new(ext_dtype: ExtDTypeRef, storage_array: ArrayRef) -> VortexResult { - let dtype = DType::Extension(ext_dtype.clone()); + vortex_ensure_eq!( + ext_dtype.storage_dtype(), + storage_array.dtype(), + "Tried to create an `ExtensionArray` with an incompatible storage array" + ); + + let dtype = DType::Extension(ext_dtype); let len = storage_array.len(); - let data = ExtensionData::try_new(ext_dtype, storage_array.dtype())?; - Ok(unsafe { - Array::from_parts_unchecked( - ArrayParts::new(Extension, dtype, len, data).with_slots(vec![Some(storage_array)]), - ) - }) + + let parts = ArrayParts::new(Extension, dtype, len, EmptyArrayData) + .with_slots(vec![Some(storage_array)]); + + Ok(unsafe { Array::from_parts_unchecked(parts) }) } /// Creates a new [`ExtensionArray`](crate::arrays::ExtensionArray) from a vtable, metadata, and @@ -176,6 +74,7 @@ impl Array { let ext_dtype = ExtDType::::try_with_vtable(vtable, metadata, storage_array.dtype().clone())? .erased(); + Self::try_new(ext_dtype, storage_array) } } diff --git a/vortex-array/src/arrays/extension/mod.rs b/vortex-array/src/arrays/extension/mod.rs index 802545025c1..57bc9df5ead 100644 --- a/vortex-array/src/arrays/extension/mod.rs +++ b/vortex-array/src/arrays/extension/mod.rs @@ -3,7 +3,6 @@ mod array; pub use array::ExtensionArrayExt; -pub use array::ExtensionData; pub use vtable::ExtensionArray; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/extension/vtable/mod.rs b/vortex-array/src/arrays/extension/vtable/mod.rs index 82ad85e701f..b048c81fff7 100644 --- a/vortex-array/src/arrays/extension/vtable/mod.rs +++ b/vortex-array/src/arrays/extension/vtable/mod.rs @@ -1,29 +1,26 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::hash::Hasher; - use kernel::PARENT_KERNELS; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; -use vortex_error::vortex_ensure; +use vortex_error::vortex_ensure_eq; +use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; use vortex_session::registry::CachedId; -use crate::ArrayEq; -use crate::ArrayHash; use crate::ArrayRef; +use crate::EmptyArrayData; use crate::ExecutionCtx; use crate::ExecutionResult; -use crate::Precision; use crate::array::Array; use crate::array::ArrayId; +use crate::array::ArrayParts; use crate::array::ArrayView; use crate::array::VTable; use crate::array::ValidityVTableFromChild; -use crate::arrays::extension::ExtensionData; use crate::arrays::extension::array::SLOT_NAMES; use crate::arrays::extension::array::STORAGE_SLOT; use crate::arrays::extension::compute::rules::PARENT_RULES; @@ -36,24 +33,52 @@ mod kernel; mod operations; mod validity; +/// An extension array that wraps another array with additional type information. +/// +/// **⚠️ Unstable API**: This is an experimental feature that may change significantly +/// in future versions. The extension type system is still evolving. +/// +/// Unlike Apache Arrow's extension arrays, Vortex extension arrays provide a more flexible +/// mechanism for adding semantic meaning to existing array types without requiring +/// changes to the core type system. +/// +/// ## Design Philosophy +/// +/// Extension arrays serve as a type-safe wrapper that: +/// - Preserves the underlying storage format and operations +/// - Adds semantic type information via `ExtDType` +/// - Enables custom serialization and deserialization logic +/// - Allows domain-specific interpretations of generic data +/// +/// ## Storage and Type Relationship +/// +/// The extension array maintains a strict contract: +/// - **Storage array**: Contains the actual data in a standard Vortex encoding +/// - **Extension type**: Defines how to interpret the storage data semantically +/// - **Type safety**: The storage array's dtype must match the extension type's storage dtype +/// +/// ## Use Cases +/// +/// Extension arrays are ideal for: +/// - **Custom numeric types**: Units of measurement, currencies +/// - **Temporal types**: Custom date/time formats, time zones, calendars +/// - **Domain-specific types**: UUIDs, IP addresses, geographic coordinates +/// - **Encoded types**: Base64 strings, compressed data, encrypted values +/// +/// ## Validity and Operations +/// +/// Extension arrays delegate validity and most operations to their storage array: +/// - Validity is inherited from the underlying storage +/// - Slicing preserves the extension type +/// - Scalar access wraps storage scalars with extension metadata #[derive(Clone, Debug)] pub struct Extension; /// A [`Extension`]-encoded Vortex array. pub type ExtensionArray = Array; -impl ArrayHash for ExtensionData { - fn array_hash(&self, _state: &mut H, _precision: Precision) {} -} - -impl ArrayEq for ExtensionData { - fn array_eq(&self, _other: &Self, _precision: Precision) -> bool { - true - } -} - impl VTable for Extension { - type ArrayData = ExtensionData; + type ArrayData = EmptyArrayData; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; @@ -65,28 +90,30 @@ impl VTable for Extension { fn validate( &self, - data: &ExtensionData, + _data: &EmptyArrayData, dtype: &DType, len: usize, slots: &[Option], ) -> VortexResult<()> { - _ = data; let storage = slots[STORAGE_SLOT] .as_ref() .vortex_expect("ExtensionArray storage slot"); - vortex_ensure!( - storage.len() == len, - "ExtensionArray length {} does not match outer length {}", + vortex_ensure_eq!( + storage.len(), + len, + "ExtensionArray length {} does not match outer length {len}", storage.len(), - len ); - let actual_dtype = DType::Extension(data.ext_dtype.clone()); - vortex_ensure!( - &actual_dtype == dtype, - "ExtensionArray dtype {} does not match outer dtype {}", - actual_dtype, - dtype + let ext_dtype = dtype + .as_extension_opt() + .ok_or_else(|| vortex_err!("not an extension dtype"))?; + + let actual_dtype = DType::Extension(ext_dtype.clone()); + vortex_ensure_eq!( + &actual_dtype, + dtype, + "ExtensionArray dtype {actual_dtype} does not match outer dtype {dtype}", ); Ok(()) @@ -120,7 +147,7 @@ impl VTable for Extension { _buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult> { + ) -> VortexResult> { if !metadata.is_empty() { vortex_bail!( "ExtensionArray expects empty metadata, got {} bytes", @@ -134,13 +161,10 @@ impl VTable for Extension { vortex_bail!("Expected 1 child, got {}", children.len()); } let storage = children.get(0, ext_dtype.storage_dtype(), len)?; - Ok(crate::array::ArrayParts::new( - self.clone(), - dtype.clone(), - len, - ExtensionData::new(ext_dtype.clone(), storage.dtype()), + Ok( + ArrayParts::new(self.clone(), dtype.clone(), len, EmptyArrayData) + .with_slots(vec![Some(storage)]), ) - .with_slots(vec![Some(storage)])) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { diff --git a/vortex-btrblocks/src/schemes/temporal.rs b/vortex-btrblocks/src/schemes/temporal.rs index 73671da0ff3..73aa9eedfb4 100644 --- a/vortex-btrblocks/src/schemes/temporal.rs +++ b/vortex-btrblocks/src/schemes/temporal.rs @@ -12,6 +12,7 @@ use vortex_array::arrays::ConstantArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::TemporalArray; +use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::dtype::extension::Matcher; use vortex_array::extension::datetime::AnyTemporal; diff --git a/vortex-tensor/src/encodings/l2_denorm.rs b/vortex-tensor/src/encodings/l2_denorm.rs index 61a72d6a4d2..6cb4fcb0626 100644 --- a/vortex-tensor/src/encodings/l2_denorm.rs +++ b/vortex-tensor/src/encodings/l2_denorm.rs @@ -5,6 +5,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_compressor::CascadingCompressor; use vortex_compressor::ctx::CompressorContext; use vortex_compressor::estimate::CompressionEstimate;