From 9ea25f0408a9684e15018f2e1293df766172e978 Mon Sep 17 00:00:00 2001 From: "Jorge C. Leitao" Date: Fri, 2 Dec 2022 05:06:36 +0000 Subject: [PATCH] Added offset module --- src/array/binary/ffi.rs | 3 ++- src/array/binary/fmt.rs | 3 ++- src/array/binary/from.rs | 2 +- src/array/binary/iterator.rs | 3 ++- src/array/binary/mod.rs | 3 ++- src/array/binary/mutable.rs | 3 ++- src/array/binary/mutable_values.rs | 4 ++-- src/array/equal/binary.rs | 3 ++- src/array/equal/list.rs | 3 ++- src/array/equal/mod.rs | 1 + src/array/equal/utf8.rs | 3 ++- src/array/growable/binary.rs | 3 ++- src/array/growable/list.rs | 3 ++- src/array/growable/utf8.rs | 3 ++- src/array/growable/utils.rs | 5 +---- src/array/list/ffi.rs | 4 +++- src/array/list/fmt.rs | 2 +- src/array/list/iterator.rs | 2 +- src/array/list/mod.rs | 3 ++- src/array/list/mutable.rs | 3 ++- src/array/mod.rs | 3 +-- src/array/ord.rs | 1 + src/array/physical_binary.rs | 2 +- src/array/utf8/ffi.rs | 3 ++- src/array/utf8/fmt.rs | 3 ++- src/array/utf8/from.rs | 2 +- src/array/utf8/iterator.rs | 3 ++- src/array/utf8/mod.rs | 3 ++- src/array/utf8/mutable.rs | 3 ++- src/array/utf8/mutable_values.rs | 3 ++- src/compute/aggregate/min_max.rs | 3 ++- src/compute/cast/binary_to.rs | 1 + src/compute/cast/boolean_to.rs | 3 ++- src/compute/cast/mod.rs | 1 + src/compute/cast/primitive_to.rs | 1 + src/compute/cast/utf8_to.rs | 1 + src/compute/comparison/binary.rs | 3 ++- src/compute/comparison/utf8.rs | 3 ++- src/compute/contains.rs | 3 ++- src/compute/hash.rs | 3 ++- src/compute/length.rs | 1 + src/compute/like.rs | 3 ++- src/compute/regex_match.rs | 6 ++++-- src/compute/sort/binary.rs | 3 ++- src/compute/sort/mod.rs | 1 + src/compute/sort/row/mod.rs | 3 ++- src/compute/sort/utf8.rs | 4 ++-- src/compute/substring.rs | 1 + src/compute/take/binary.rs | 3 ++- src/compute/take/generic_binary.rs | 3 ++- src/compute/take/list.rs | 3 ++- src/compute/take/utf8.rs | 3 ++- src/compute/utf8.rs | 3 ++- src/ffi/mmap.rs | 3 ++- src/io/avro/read/nested.rs | 1 + src/io/avro/write/serialize.rs | 1 + src/io/csv/read_utils.rs | 15 ++++++++------- src/io/csv/write/serialize.rs | 6 ++++-- src/io/ipc/read/array/binary.rs | 3 ++- src/io/ipc/read/array/list.rs | 3 ++- src/io/ipc/read/array/utf8.rs | 3 ++- src/io/ipc/write/serialize.rs | 3 ++- src/io/json/read/deserialize.rs | 1 + src/io/json/write/serialize.rs | 1 + src/io/json_integration/read/array.rs | 1 + src/io/odbc/write/serialize.rs | 1 + src/io/orc/read/mod.rs | 5 ++--- src/io/parquet/read/deserialize/binary/basic.rs | 3 ++- .../parquet/read/deserialize/binary/dictionary.rs | 3 ++- src/io/parquet/read/deserialize/binary/nested.rs | 4 ++-- src/io/parquet/read/deserialize/binary/utils.rs | 2 +- src/io/parquet/read/statistics/binary.rs | 3 ++- src/io/parquet/read/statistics/utf8.rs | 3 ++- src/io/parquet/write/binary/basic.rs | 3 ++- src/io/parquet/write/binary/nested.rs | 3 ++- src/io/parquet/write/nested/def.rs | 2 +- src/io/parquet/write/nested/mod.rs | 2 +- src/io/parquet/write/pages.rs | 3 ++- src/io/parquet/write/utf8/basic.rs | 3 ++- src/io/parquet/write/utf8/nested.rs | 3 ++- src/lib.rs | 1 + src/offset.rs | 2 ++ src/scalar/binary.rs | 2 +- src/scalar/list.rs | 2 +- src/scalar/utf8.rs | 2 +- src/temporal_conversions.rs | 3 ++- src/util/bench_util.rs | 2 +- tests/it/array/equal/utf8.rs | 1 + tests/it/compute/length.rs | 1 + tests/it/compute/regex_match.rs | 3 ++- tests/it/compute/substring.rs | 2 +- tests/it/compute/utf8.rs | 2 +- tests/it/io/parquet/mod.rs | 1 + 93 files changed, 162 insertions(+), 90 deletions(-) create mode 100644 src/offset.rs diff --git a/src/array/binary/ffi.rs b/src/array/binary/ffi.rs index 2c6792237ad..f592773f56e 100644 --- a/src/array/binary/ffi.rs +++ b/src/array/binary/ffi.rs @@ -1,7 +1,8 @@ use crate::{ - array::{FromFfi, Offset, ToFfi}, + array::{FromFfi, ToFfi}, bitmap::align, ffi, + offset::Offset, }; use crate::error::Result; diff --git a/src/array/binary/fmt.rs b/src/array/binary/fmt.rs index c068d0d3656..c2ec8737fb2 100644 --- a/src/array/binary/fmt.rs +++ b/src/array/binary/fmt.rs @@ -1,7 +1,8 @@ use std::fmt::{Debug, Formatter, Result, Write}; +use crate::offset::Offset; + use super::super::fmt::write_vec; -use super::super::Offset; use super::BinaryArray; pub fn write_value(array: &BinaryArray, index: usize, f: &mut W) -> Result { diff --git a/src/array/binary/from.rs b/src/array/binary/from.rs index aa575ccd9cc..8556da6906f 100644 --- a/src/array/binary/from.rs +++ b/src/array/binary/from.rs @@ -1,6 +1,6 @@ use std::iter::FromIterator; -use crate::array::Offset; +use crate::offset::Offset; use super::{BinaryArray, MutableBinaryArray}; diff --git a/src/array/binary/iterator.rs b/src/array/binary/iterator.rs index 2af79e6a296..042913a71d9 100644 --- a/src/array/binary/iterator.rs +++ b/src/array/binary/iterator.rs @@ -1,6 +1,7 @@ use crate::{ - array::{ArrayAccessor, ArrayValuesIter, Offset}, + array::{ArrayAccessor, ArrayValuesIter}, bitmap::utils::{BitmapIter, ZipValidity}, + offset::Offset, }; use super::{BinaryArray, MutableBinaryValuesArray}; diff --git a/src/array/binary/mod.rs b/src/array/binary/mod.rs index e7a4e9fe8c0..612d988215e 100644 --- a/src/array/binary/mod.rs +++ b/src/array/binary/mod.rs @@ -6,6 +6,7 @@ use crate::{ buffer::Buffer, datatypes::DataType, error::Error, + offset::Offset, trusted_len::TrustedLen, }; @@ -13,7 +14,7 @@ use either::Either; use super::{ specification::{try_check_offsets, try_check_offsets_bounds}, - Array, GenericBinaryArray, Offset, + Array, GenericBinaryArray, }; mod ffi; diff --git a/src/array/binary/mutable.rs b/src/array/binary/mutable.rs index 15035aed0ed..36f31eee5a5 100644 --- a/src/array/binary/mutable.rs +++ b/src/array/binary/mutable.rs @@ -1,13 +1,14 @@ use std::{iter::FromIterator, sync::Arc}; use crate::{ - array::{Array, MutableArray, Offset, TryExtend, TryExtendFromSelf, TryPush}, + array::{Array, MutableArray, TryExtend, TryExtendFromSelf, TryPush}, bitmap::{ utils::{BitmapIter, ZipValidity}, Bitmap, MutableBitmap, }, datatypes::DataType, error::{Error, Result}, + offset::Offset, trusted_len::TrustedLen, }; diff --git a/src/array/binary/mutable_values.rs b/src/array/binary/mutable_values.rs index 53a43d69c7a..1d608b7403f 100644 --- a/src/array/binary/mutable_values.rs +++ b/src/array/binary/mutable_values.rs @@ -3,12 +3,12 @@ use std::{iter::FromIterator, sync::Arc}; use crate::{ array::{ specification::{check_offsets_minimal, try_check_offsets}, - Array, ArrayAccessor, ArrayValuesIter, MutableArray, Offset, TryExtend, TryExtendFromSelf, - TryPush, + Array, ArrayAccessor, ArrayValuesIter, MutableArray, TryExtend, TryExtendFromSelf, TryPush, }, bitmap::MutableBitmap, datatypes::DataType, error::{Error, Result}, + offset::Offset, trusted_len::TrustedLen, }; diff --git a/src/array/equal/binary.rs b/src/array/equal/binary.rs index 1c86fab6dce..bed8588efb5 100644 --- a/src/array/equal/binary.rs +++ b/src/array/equal/binary.rs @@ -1,4 +1,5 @@ -use crate::array::{BinaryArray, Offset}; +use crate::array::BinaryArray; +use crate::offset::Offset; pub(super) fn equal(lhs: &BinaryArray, rhs: &BinaryArray) -> bool { lhs.data_type() == rhs.data_type() && lhs.len() == rhs.len() && lhs.iter().eq(rhs.iter()) diff --git a/src/array/equal/list.rs b/src/array/equal/list.rs index 4eee0b821e2..26faa1598fa 100644 --- a/src/array/equal/list.rs +++ b/src/array/equal/list.rs @@ -1,4 +1,5 @@ -use crate::array::{Array, ListArray, Offset}; +use crate::array::{Array, ListArray}; +use crate::offset::Offset; pub(super) fn equal(lhs: &ListArray, rhs: &ListArray) -> bool { lhs.data_type() == rhs.data_type() && lhs.len() == rhs.len() && lhs.iter().eq(rhs.iter()) diff --git a/src/array/equal/mod.rs b/src/array/equal/mod.rs index aa2ea602882..2bb3ba77f1f 100644 --- a/src/array/equal/mod.rs +++ b/src/array/equal/mod.rs @@ -1,3 +1,4 @@ +use crate::offset::Offset; use crate::types::NativeType; use super::*; diff --git a/src/array/equal/utf8.rs b/src/array/equal/utf8.rs index 3a8f0e5f012..1327221ca33 100644 --- a/src/array/equal/utf8.rs +++ b/src/array/equal/utf8.rs @@ -1,4 +1,5 @@ -use crate::array::{Offset, Utf8Array}; +use crate::array::Utf8Array; +use crate::offset::Offset; pub(super) fn equal(lhs: &Utf8Array, rhs: &Utf8Array) -> bool { lhs.data_type() == rhs.data_type() && lhs.len() == rhs.len() && lhs.iter().eq(rhs.iter()) diff --git a/src/array/growable/binary.rs b/src/array/growable/binary.rs index c3403877e9e..fd91590a25d 100644 --- a/src/array/growable/binary.rs +++ b/src/array/growable/binary.rs @@ -1,9 +1,10 @@ use std::sync::Arc; use crate::{ - array::{Array, BinaryArray, Offset}, + array::{Array, BinaryArray}, bitmap::MutableBitmap, datatypes::DataType, + offset::Offset, }; use super::{ diff --git a/src/array/growable/list.rs b/src/array/growable/list.rs index 0e1e7ceb5f8..3fcbe3c4539 100644 --- a/src/array/growable/list.rs +++ b/src/array/growable/list.rs @@ -1,8 +1,9 @@ use std::sync::Arc; use crate::{ - array::{Array, ListArray, Offset}, + array::{Array, ListArray}, bitmap::MutableBitmap, + offset::Offset, }; use super::{ diff --git a/src/array/growable/utf8.rs b/src/array/growable/utf8.rs index adfdba2cd53..eed8ba30159 100644 --- a/src/array/growable/utf8.rs +++ b/src/array/growable/utf8.rs @@ -1,7 +1,8 @@ use std::sync::Arc; use crate::{ - array::{Array, Offset, Utf8Array}, + array::{Array, Utf8Array}, + offset::Offset, bitmap::MutableBitmap, }; diff --git a/src/array/growable/utils.rs b/src/array/growable/utils.rs index 7e39df295de..d06c1116d48 100644 --- a/src/array/growable/utils.rs +++ b/src/array/growable/utils.rs @@ -1,7 +1,4 @@ -use crate::{ - array::{Array, Offset}, - bitmap::MutableBitmap, -}; +use crate::{array::Array, bitmap::MutableBitmap, offset::Offset}; pub(super) fn extend_offsets(buffer: &mut Vec, last_offset: &mut T, offsets: &[T]) { buffer.reserve(offsets.len() - 1); diff --git a/src/array/list/ffi.rs b/src/array/list/ffi.rs index c6f1b2985dd..9d0b19a85e0 100644 --- a/src/array/list/ffi.rs +++ b/src/array/list/ffi.rs @@ -1,6 +1,8 @@ use crate::{array::FromFfi, bitmap::align, error::Result, ffi}; -use super::super::{ffi::ToFfi, Array, Offset}; +use crate::offset::Offset; + +use super::super::{ffi::ToFfi, Array}; use super::ListArray; unsafe impl ToFfi for ListArray { diff --git a/src/array/list/fmt.rs b/src/array/list/fmt.rs index e6103ded6cb..4b10fc8f936 100644 --- a/src/array/list/fmt.rs +++ b/src/array/list/fmt.rs @@ -1,6 +1,6 @@ use std::fmt::{Debug, Formatter, Result, Write}; -use crate::array::Offset; +use crate::offset::Offset; use super::super::fmt::{get_display, write_vec}; use super::ListArray; diff --git a/src/array/list/iterator.rs b/src/array/list/iterator.rs index 82b5c7dca5f..86a12dfe769 100644 --- a/src/array/list/iterator.rs +++ b/src/array/list/iterator.rs @@ -1,6 +1,6 @@ -use crate::array::Offset; use crate::array::{Array, ArrayAccessor, ArrayValuesIter}; use crate::bitmap::utils::{BitmapIter, ZipValidity}; +use crate::offset::Offset; use super::ListArray; diff --git a/src/array/list/mod.rs b/src/array/list/mod.rs index feb5a5df93b..7740307799d 100644 --- a/src/array/list/mod.rs +++ b/src/array/list/mod.rs @@ -3,13 +3,14 @@ use crate::{ buffer::Buffer, datatypes::{DataType, Field}, error::Error, + offset::Offset, }; use std::sync::Arc; use super::{ new_empty_array, specification::{try_check_offsets, try_check_offsets_bounds}, - Array, Offset, + Array, }; mod ffi; diff --git a/src/array/list/mutable.rs b/src/array/list/mutable.rs index 57017998d0b..97785a01740 100644 --- a/src/array/list/mutable.rs +++ b/src/array/list/mutable.rs @@ -4,11 +4,12 @@ use crate::{ array::{ physical_binary::{extend_validity, try_extend_offsets}, specification::try_check_offsets, - Array, MutableArray, Offset, TryExtend, TryExtendFromSelf, TryPush, + Array, MutableArray, TryExtend, TryExtendFromSelf, TryPush, }, bitmap::MutableBitmap, datatypes::{DataType, Field}, error::{Error, Result}, + offset::Offset, trusted_len::TrustedLen, }; diff --git a/src/array/mod.rs b/src/array/mod.rs index 45e83a80803..5aa1dd1eb64 100644 --- a/src/array/mod.rs +++ b/src/array/mod.rs @@ -439,7 +439,6 @@ pub use iterator::ArrayValuesIter; pub use equal::equal; pub use fmt::{get_display, get_value_display}; -pub use crate::types::Offset; pub use binary::{BinaryArray, BinaryValueIter, MutableBinaryArray, MutableBinaryValuesArray}; pub use boolean::{BooleanArray, MutableBooleanArray}; pub use dictionary::{DictionaryArray, DictionaryKey, MutableDictionaryArray}; @@ -483,7 +482,7 @@ pub trait TryExtendFromSelf { /// 1. `offsets.len() > 0` /// 2. `offsets[i] >= offsets[i-1] for all i` /// 3. `offsets[i] < values.len() for all i` -pub unsafe trait GenericBinaryArray: Array { +pub unsafe trait GenericBinaryArray: Array { /// The values of the array fn values(&self) -> &[u8]; /// The offsets of the array diff --git a/src/array/ord.rs b/src/array/ord.rs index 639317165ab..e42c7fa0fc7 100644 --- a/src/array/ord.rs +++ b/src/array/ord.rs @@ -4,6 +4,7 @@ use std::cmp::Ordering; use crate::datatypes::*; use crate::error::{Error, Result}; +use crate::offset::Offset; use crate::{array::*, types::NativeType}; /// Compare the values at two arbitrary indices in two arrays. diff --git a/src/array/physical_binary.rs b/src/array/physical_binary.rs index bf883f4e6e8..825ba01e5d0 100644 --- a/src/array/physical_binary.rs +++ b/src/array/physical_binary.rs @@ -1,6 +1,6 @@ -use crate::array::Offset; use crate::bitmap::MutableBitmap; use crate::error::Error; +use crate::offset::Offset; /// # Safety /// The caller must ensure that `iterator` is `TrustedLen`. diff --git a/src/array/utf8/ffi.rs b/src/array/utf8/ffi.rs index abaed6b28e7..2152dfade45 100644 --- a/src/array/utf8/ffi.rs +++ b/src/array/utf8/ffi.rs @@ -1,8 +1,9 @@ use crate::{ - array::{FromFfi, Offset, ToFfi}, + array::{FromFfi, ToFfi}, bitmap::align, error::Result, ffi, + offset::Offset, }; use super::Utf8Array; diff --git a/src/array/utf8/fmt.rs b/src/array/utf8/fmt.rs index 6ea28feae12..1b6868c4a0e 100644 --- a/src/array/utf8/fmt.rs +++ b/src/array/utf8/fmt.rs @@ -1,7 +1,8 @@ use std::fmt::{Debug, Formatter, Result, Write}; +use crate::offset::Offset; + use super::super::fmt::write_vec; -use super::super::Offset; use super::Utf8Array; pub fn write_value(array: &Utf8Array, index: usize, f: &mut W) -> Result { diff --git a/src/array/utf8/from.rs b/src/array/utf8/from.rs index 1a0a0a1f7e2..f6866998312 100644 --- a/src/array/utf8/from.rs +++ b/src/array/utf8/from.rs @@ -1,6 +1,6 @@ use std::iter::FromIterator; -use crate::array::Offset; +use crate::offset::Offset; use super::{MutableUtf8Array, Utf8Array}; diff --git a/src/array/utf8/iterator.rs b/src/array/utf8/iterator.rs index 071fc54d8ae..2a5ba87c3e8 100644 --- a/src/array/utf8/iterator.rs +++ b/src/array/utf8/iterator.rs @@ -1,5 +1,6 @@ -use crate::array::{ArrayAccessor, ArrayValuesIter, Offset}; +use crate::array::{ArrayAccessor, ArrayValuesIter}; use crate::bitmap::utils::{BitmapIter, ZipValidity}; +use crate::offset::Offset; use super::{MutableUtf8Array, MutableUtf8ValuesArray, Utf8Array}; diff --git a/src/array/utf8/mod.rs b/src/array/utf8/mod.rs index f91e1466451..f011183ce6e 100644 --- a/src/array/utf8/mod.rs +++ b/src/array/utf8/mod.rs @@ -6,6 +6,7 @@ use crate::{ buffer::Buffer, datatypes::DataType, error::{Error, Result}, + offset::Offset, trusted_len::TrustedLen, }; @@ -13,7 +14,7 @@ use either::Either; use super::{ specification::{try_check_offsets_and_utf8, try_check_offsets_bounds}, - Array, GenericBinaryArray, Offset, + Array, GenericBinaryArray, }; mod ffi; diff --git a/src/array/utf8/mutable.rs b/src/array/utf8/mutable.rs index 27d28dc64b2..4dc9b1304b8 100644 --- a/src/array/utf8/mutable.rs +++ b/src/array/utf8/mutable.rs @@ -2,13 +2,14 @@ use std::{iter::FromIterator, sync::Arc}; use crate::array::{physical_binary::*, TryExtendFromSelf}; use crate::{ - array::{Array, MutableArray, Offset, TryExtend, TryPush}, + array::{Array, MutableArray, TryExtend, TryPush}, bitmap::{ utils::{BitmapIter, ZipValidity}, Bitmap, MutableBitmap, }, datatypes::DataType, error::{Error, Result}, + offset::Offset, trusted_len::TrustedLen, }; diff --git a/src/array/utf8/mutable_values.rs b/src/array/utf8/mutable_values.rs index 5e33144452f..0da7ff8ff7c 100644 --- a/src/array/utf8/mutable_values.rs +++ b/src/array/utf8/mutable_values.rs @@ -3,11 +3,12 @@ use std::{iter::FromIterator, sync::Arc}; use crate::{ array::{ specification::{check_offsets_minimal, try_check_offsets_and_utf8}, - Array, ArrayValuesIter, MutableArray, Offset, TryExtend, TryExtendFromSelf, TryPush, + Array, ArrayValuesIter, MutableArray, TryExtend, TryExtendFromSelf, TryPush, }, bitmap::MutableBitmap, datatypes::DataType, error::{Error, Result}, + offset::Offset, trusted_len::TrustedLen, }; diff --git a/src/compute/aggregate/min_max.rs b/src/compute/aggregate/min_max.rs index 00f76b052bc..886cd509dd6 100644 --- a/src/compute/aggregate/min_max.rs +++ b/src/compute/aggregate/min_max.rs @@ -1,11 +1,12 @@ use crate::bitmap::utils::{BitChunkIterExact, BitChunksExact}; use crate::datatypes::{DataType, PhysicalType, PrimitiveType}; use crate::error::{Error, Result}; +use crate::offset::Offset; use crate::scalar::*; use crate::types::simd::*; use crate::types::NativeType; use crate::{ - array::{Array, BinaryArray, BooleanArray, Offset, PrimitiveArray, Utf8Array}, + array::{Array, BinaryArray, BooleanArray, PrimitiveArray, Utf8Array}, bitmap::Bitmap, }; diff --git a/src/compute/cast/binary_to.rs b/src/compute/cast/binary_to.rs index 44224321b55..d4b63c1e73d 100644 --- a/src/compute/cast/binary_to.rs +++ b/src/compute/cast/binary_to.rs @@ -1,6 +1,7 @@ use std::convert::TryFrom; use crate::error::{Error, Result}; +use crate::offset::Offset; use crate::{array::*, datatypes::DataType, types::NativeType}; use super::CastOptions; diff --git a/src/compute/cast/boolean_to.rs b/src/compute/cast/boolean_to.rs index ef24e4b4dff..1ce45c87118 100644 --- a/src/compute/cast/boolean_to.rs +++ b/src/compute/cast/boolean_to.rs @@ -1,6 +1,7 @@ use crate::{ - array::{Array, BinaryArray, BooleanArray, Offset, PrimitiveArray, Utf8Array}, + array::{Array, BinaryArray, BooleanArray, PrimitiveArray, Utf8Array}, error::Result, + offset::Offset, types::NativeType, }; diff --git a/src/compute/cast/mod.rs b/src/compute/cast/mod.rs index edb6c223568..eda7bda78a0 100644 --- a/src/compute/cast/mod.rs +++ b/src/compute/cast/mod.rs @@ -18,6 +18,7 @@ use crate::{ array::*, datatypes::*, error::{Error, Result}, + offset::Offset, }; /// options defining how Cast kernels behave diff --git a/src/compute/cast/primitive_to.rs b/src/compute/cast/primitive_to.rs index a36f16e3deb..6c5c7bb8753 100644 --- a/src/compute/cast/primitive_to.rs +++ b/src/compute/cast/primitive_to.rs @@ -4,6 +4,7 @@ use num_traits::{AsPrimitive, Float, ToPrimitive}; use crate::datatypes::IntervalUnit; use crate::error::Result; +use crate::offset::Offset; use crate::types::{days_ms, f16, months_days_ns}; use crate::{ array::*, diff --git a/src/compute/cast/utf8_to.rs b/src/compute/cast/utf8_to.rs index 996889174a2..2625497ed67 100644 --- a/src/compute/cast/utf8_to.rs +++ b/src/compute/cast/utf8_to.rs @@ -6,6 +6,7 @@ use crate::{ array::*, datatypes::DataType, error::{Error, Result}, + offset::Offset, temporal_conversions::{ utf8_to_naive_timestamp_ns as utf8_to_naive_timestamp_ns_, utf8_to_timestamp_ns as utf8_to_timestamp_ns_, EPOCH_DAYS_FROM_CE, diff --git a/src/compute/comparison/binary.rs b/src/compute/comparison/binary.rs index 0317a9b7783..6787e184618 100644 --- a/src/compute/comparison/binary.rs +++ b/src/compute/comparison/binary.rs @@ -1,9 +1,10 @@ //! Comparison functions for [`BinaryArray`] use crate::compute::comparison::{finish_eq_validities, finish_neq_validities}; use crate::{ - array::{BinaryArray, BooleanArray, Offset}, + array::{BinaryArray, BooleanArray}, bitmap::Bitmap, datatypes::DataType, + offset::Offset, }; use super::super::utils::combine_validities; diff --git a/src/compute/comparison/utf8.rs b/src/compute/comparison/utf8.rs index 31983fa816c..05d84d803ff 100644 --- a/src/compute/comparison/utf8.rs +++ b/src/compute/comparison/utf8.rs @@ -1,9 +1,10 @@ //! Comparison functions for [`Utf8Array`] use crate::compute::comparison::{finish_eq_validities, finish_neq_validities}; use crate::{ - array::{BooleanArray, Offset, Utf8Array}, + array::{BooleanArray, Utf8Array}, bitmap::Bitmap, datatypes::DataType, + offset::Offset, }; use super::super::utils::combine_validities; diff --git a/src/compute/contains.rs b/src/compute/contains.rs index a1ede1dd29d..738a8e11e57 100644 --- a/src/compute/contains.rs +++ b/src/compute/contains.rs @@ -1,10 +1,11 @@ //! Declares the [`contains`] operator use crate::{ - array::{Array, BinaryArray, BooleanArray, ListArray, Offset, PrimitiveArray, Utf8Array}, + array::{Array, BinaryArray, BooleanArray, ListArray, PrimitiveArray, Utf8Array}, bitmap::Bitmap, datatypes::DataType, error::{Error, Result}, + offset::Offset, types::NativeType, }; diff --git a/src/compute/hash.rs b/src/compute/hash.rs index 68232b4d015..d5f8370d8b4 100644 --- a/src/compute/hash.rs +++ b/src/compute/hash.rs @@ -12,9 +12,10 @@ macro_rules! new_state { } use crate::{ - array::{Array, BinaryArray, BooleanArray, Offset, PrimitiveArray, Utf8Array}, + array::{Array, BinaryArray, BooleanArray, PrimitiveArray, Utf8Array}, datatypes::{DataType, PhysicalType, PrimitiveType}, error::{Error, Result}, + offset::Offset, types::NativeType, }; diff --git a/src/compute/length.rs b/src/compute/length.rs index c52541a8917..8cea7232894 100644 --- a/src/compute/length.rs +++ b/src/compute/length.rs @@ -21,6 +21,7 @@ use crate::{ array::*, datatypes::DataType, error::{Error, Result}, + offset::Offset, types::NativeType, }; diff --git a/src/compute/like.rs b/src/compute/like.rs index bf363972a1a..5c736cd8b5f 100644 --- a/src/compute/like.rs +++ b/src/compute/like.rs @@ -5,11 +5,12 @@ use regex::bytes::Regex as BytesRegex; use regex::Regex; use crate::{ - array::{BinaryArray, BooleanArray, Offset, Utf8Array}, + array::{BinaryArray, BooleanArray, Utf8Array}, bitmap::Bitmap, compute::utils::combine_validities, datatypes::DataType, error::{Error, Result}, + offset::Offset, }; #[inline] diff --git a/src/compute/regex_match.rs b/src/compute/regex_match.rs index 41cacb293bf..371eb073e46 100644 --- a/src/compute/regex_match.rs +++ b/src/compute/regex_match.rs @@ -3,11 +3,13 @@ use ahash::AHashMap; use regex::Regex; -use super::utils::combine_validities; -use crate::array::{BooleanArray, Offset, Utf8Array}; +use crate::array::{BooleanArray, Utf8Array}; use crate::bitmap::Bitmap; use crate::datatypes::DataType; use crate::error::{Error, Result}; +use crate::offset::Offset; + +use super::utils::combine_validities; /// Regex matches pub fn regex_match(values: &Utf8Array, regex: &Utf8Array) -> Result { diff --git a/src/compute/sort/binary.rs b/src/compute/sort/binary.rs index 766efc678b2..cf0992b4b99 100644 --- a/src/compute/sort/binary.rs +++ b/src/compute/sort/binary.rs @@ -1,4 +1,5 @@ -use crate::array::{BinaryArray, Offset, PrimitiveArray}; +use crate::array::{BinaryArray, PrimitiveArray}; +use crate::offset::Offset; use crate::types::Index; use super::common; diff --git a/src/compute/sort/mod.rs b/src/compute/sort/mod.rs index be0f15a1ae2..5394e370280 100644 --- a/src/compute/sort/mod.rs +++ b/src/compute/sort/mod.rs @@ -5,6 +5,7 @@ use crate::array::ord; use crate::compute::take; use crate::datatypes::*; use crate::error::{Error, Result}; +use crate::offset::Offset; use crate::{ array::*, types::{Index, NativeType}, diff --git a/src/compute/sort/row/mod.rs b/src/compute/sort/row/mod.rs index 8d4833af089..005e046fc92 100644 --- a/src/compute/sort/row/mod.rs +++ b/src/compute/sort/row/mod.rs @@ -637,9 +637,10 @@ mod tests { use super::*; use crate::{ - array::{Array, DictionaryKey, Float32Array, Int16Array, NullArray, Offset}, + array::{Array, DictionaryKey, Float32Array, Int16Array, NullArray}, compute::sort::build_compare, datatypes::DataType, + offset::Offset, types::NativeType, }; diff --git a/src/compute/sort/utf8.rs b/src/compute/sort/utf8.rs index e2e2da1bc56..0d7190eb23f 100644 --- a/src/compute/sort/utf8.rs +++ b/src/compute/sort/utf8.rs @@ -1,5 +1,5 @@ -use crate::array::{DictionaryArray, DictionaryKey}; -use crate::array::{Offset, PrimitiveArray, Utf8Array}; +use crate::array::{DictionaryArray, DictionaryKey, PrimitiveArray, Utf8Array}; +use crate::offset::Offset; use crate::types::Index; use super::common; diff --git a/src/compute/substring.rs b/src/compute/substring.rs index d879a971707..1edab3a0cb1 100644 --- a/src/compute/substring.rs +++ b/src/compute/substring.rs @@ -21,6 +21,7 @@ use crate::{ array::*, datatypes::DataType, error::{Error, Result}, + offset::Offset, }; fn utf8_substring(array: &Utf8Array, start: O, length: &Option) -> Utf8Array { diff --git a/src/compute/take/binary.rs b/src/compute/take/binary.rs index 0651b6bba13..4d116d650df 100644 --- a/src/compute/take/binary.rs +++ b/src/compute/take/binary.rs @@ -15,7 +15,8 @@ // specific language governing permissions and limitations // under the License. -use crate::array::{Array, BinaryArray, Offset, PrimitiveArray}; +use crate::array::{Array, BinaryArray, PrimitiveArray}; +use crate::offset::Offset; use super::generic_binary::*; use super::Index; diff --git a/src/compute/take/generic_binary.rs b/src/compute/take/generic_binary.rs index ef78184f6b8..3656aebc771 100644 --- a/src/compute/take/generic_binary.rs +++ b/src/compute/take/generic_binary.rs @@ -1,7 +1,8 @@ use crate::{ - array::{GenericBinaryArray, Offset, PrimitiveArray}, + array::{GenericBinaryArray, PrimitiveArray}, bitmap::{Bitmap, MutableBitmap}, buffer::Buffer, + offset::Offset, }; use super::Index; diff --git a/src/compute/take/list.rs b/src/compute/take/list.rs index 5e8b1d10e7c..6abc1d10155 100644 --- a/src/compute/take/list.rs +++ b/src/compute/take/list.rs @@ -17,8 +17,9 @@ use crate::array::{ growable::{Growable, GrowableList}, - ListArray, Offset, PrimitiveArray, + ListArray, PrimitiveArray, }; +use crate::offset::Offset; use super::Index; diff --git a/src/compute/take/utf8.rs b/src/compute/take/utf8.rs index 99fc091f491..490e76bf4b8 100644 --- a/src/compute/take/utf8.rs +++ b/src/compute/take/utf8.rs @@ -15,7 +15,8 @@ // specific language governing permissions and limitations // under the License. -use crate::array::{Array, Offset, PrimitiveArray, Utf8Array}; +use crate::array::{Array, PrimitiveArray, Utf8Array}; +use crate::offset::Offset; use super::generic_binary::*; use super::Index; diff --git a/src/compute/utf8.rs b/src/compute/utf8.rs index 37c1c6ff47e..2e480016ef5 100644 --- a/src/compute/utf8.rs +++ b/src/compute/utf8.rs @@ -1,9 +1,10 @@ //! Defines common maps to a [`Utf8Array`] use crate::{ - array::{Array, Offset, Utf8Array}, + array::{Array, Utf8Array}, datatypes::DataType, error::{Error, Result}, + offset::Offset, }; /// utf8_apply will apply `Fn(&str) -> String` to every value in Utf8Array. diff --git a/src/ffi/mmap.rs b/src/ffi/mmap.rs index 4bd006393f7..32be7d763eb 100644 --- a/src/ffi/mmap.rs +++ b/src/ffi/mmap.rs @@ -1,9 +1,10 @@ use std::collections::VecDeque; use std::sync::Arc; -use crate::array::{Array, DictionaryKey, FixedSizeListArray, ListArray, Offset, StructArray}; +use crate::array::{Array, DictionaryKey, FixedSizeListArray, ListArray, StructArray}; use crate::datatypes::DataType; use crate::error::Error; +use crate::offset::Offset; use crate::io::ipc::read::{Dictionaries, OutOfSpecKind}; use crate::io::ipc::read::{IpcBuffer, Node}; diff --git a/src/io/avro/read/nested.rs b/src/io/avro/read/nested.rs index 649a702ff9c..04d9bcf43b6 100644 --- a/src/io/avro/read/nested.rs +++ b/src/io/avro/read/nested.rs @@ -2,6 +2,7 @@ use crate::array::*; use crate::bitmap::*; use crate::datatypes::*; use crate::error::*; +use crate::offset::Offset; /// Auxiliary struct #[derive(Debug)] diff --git a/src/io/avro/write/serialize.rs b/src/io/avro/write/serialize.rs index cb94b78d28b..5310400ffb8 100644 --- a/src/io/avro/write/serialize.rs +++ b/src/io/avro/write/serialize.rs @@ -3,6 +3,7 @@ use avro_schema::write::encode; use crate::bitmap::utils::ZipValidity; use crate::datatypes::{IntervalUnit, PhysicalType, PrimitiveType}; +use crate::offset::Offset; use crate::types::months_days_ns; use crate::{array::*, datatypes::DataType}; diff --git a/src/io/csv/read_utils.rs b/src/io/csv/read_utils.rs index 27bdcff163d..278dd32cb21 100644 --- a/src/io/csv/read_utils.rs +++ b/src/io/csv/read_utils.rs @@ -1,23 +1,24 @@ use chrono::Datelike; -// Ideally this trait should not be needed and both `csv` and `csv_async` crates would share -// the same `ByteRecord` struct. Unfortunately, they do not and thus we must use generics -// over this trait and materialize the generics for each struct. -pub(crate) trait ByteRecordGeneric { - fn get(&self, index: usize) -> Option<&[u8]>; -} - use crate::{ array::*, chunk::Chunk, datatypes::*, error::{Error, Result}, + offset::Offset, temporal_conversions, types::NativeType, }; use super::utils::RFC3339; +// Ideally this trait should not be needed and both `csv` and `csv_async` crates would share +// the same `ByteRecord` struct. Unfortunately, they do not and thus we must use generics +// over this trait and materialize the generics for each struct. +pub(crate) trait ByteRecordGeneric { + fn get(&self, index: usize) -> Option<&[u8]>; +} + #[inline] fn to_utf8(bytes: &[u8]) -> Option<&str> { simdutf8::basic::from_utf8(bytes).ok() diff --git a/src/io/csv/write/serialize.rs b/src/io/csv/write/serialize.rs index 6f704d1cc08..46addc378f0 100644 --- a/src/io/csv/write/serialize.rs +++ b/src/io/csv/write/serialize.rs @@ -5,13 +5,15 @@ use crate::temporal_conversions; use crate::types::NativeType; use crate::util::lexical_to_bytes_mut; use crate::{ - array::{Array, BinaryArray, BooleanArray, PrimitiveArray, Utf8Array}, + array::{ + Array, BinaryArray, BooleanArray, DictionaryArray, DictionaryKey, PrimitiveArray, Utf8Array, + }, datatypes::{DataType, TimeUnit}, error::Result, + offset::Offset, }; use super::super::super::iterator::{BufStreamingIterator, StreamingIterator}; -use crate::array::{DictionaryArray, DictionaryKey, Offset}; use csv_core::WriteResult; use std::any::Any; use std::fmt::{Debug, Write}; diff --git a/src/io/ipc/read/array/binary.rs b/src/io/ipc/read/array/binary.rs index 3ee6c4ba288..eea120a5b2f 100644 --- a/src/io/ipc/read/array/binary.rs +++ b/src/io/ipc/read/array/binary.rs @@ -1,10 +1,11 @@ use std::collections::VecDeque; use std::io::{Read, Seek}; -use crate::array::{BinaryArray, Offset}; +use crate::array::BinaryArray; use crate::buffer::Buffer; use crate::datatypes::DataType; use crate::error::{Error, Result}; +use crate::offset::Offset; use super::super::read_basic::*; use super::super::{Compression, IpcBuffer, Node, OutOfSpecKind}; diff --git a/src/io/ipc/read/array/list.rs b/src/io/ipc/read/array/list.rs index 7870b85a7b1..8824ed86fe1 100644 --- a/src/io/ipc/read/array/list.rs +++ b/src/io/ipc/read/array/list.rs @@ -2,10 +2,11 @@ use std::collections::VecDeque; use std::convert::TryInto; use std::io::{Read, Seek}; -use crate::array::{ListArray, Offset}; +use crate::array::ListArray; use crate::buffer::Buffer; use crate::datatypes::DataType; use crate::error::{Error, Result}; +use crate::offset::Offset; use super::super::super::IpcField; use super::super::deserialize::{read, skip}; diff --git a/src/io/ipc/read/array/utf8.rs b/src/io/ipc/read/array/utf8.rs index f57e51ecc5a..1ff056d6f8c 100644 --- a/src/io/ipc/read/array/utf8.rs +++ b/src/io/ipc/read/array/utf8.rs @@ -1,10 +1,11 @@ use std::collections::VecDeque; use std::io::{Read, Seek}; -use crate::array::{Offset, Utf8Array}; +use crate::array::Utf8Array; use crate::buffer::Buffer; use crate::datatypes::DataType; use crate::error::{Error, Result}; +use crate::offset::Offset; use super::super::read_basic::*; use super::super::{Compression, IpcBuffer, Node, OutOfSpecKind}; diff --git a/src/io/ipc/write/serialize.rs b/src/io/ipc/write/serialize.rs index 28647bd9111..7737cbca9cd 100644 --- a/src/io/ipc/write/serialize.rs +++ b/src/io/ipc/write/serialize.rs @@ -2,7 +2,8 @@ use arrow_format::ipc; use crate::{ - array::*, bitmap::Bitmap, datatypes::PhysicalType, trusted_len::TrustedLen, types::NativeType, + array::*, bitmap::Bitmap, datatypes::PhysicalType, offset::Offset, trusted_len::TrustedLen, + types::NativeType, }; use super::super::compression; diff --git a/src/io/json/read/deserialize.rs b/src/io/json/read/deserialize.rs index ea5d25a0a5d..1d3997b1da1 100644 --- a/src/io/json/read/deserialize.rs +++ b/src/io/json/read/deserialize.rs @@ -12,6 +12,7 @@ use crate::{ chunk::Chunk, datatypes::{DataType, Field, IntervalUnit, Schema}, error::Error, + offset::Offset, types::{f16, NativeType}, }; diff --git a/src/io/json/write/serialize.rs b/src/io/json/write/serialize.rs index aa323109e9b..9bf08fc8968 100644 --- a/src/io/json/write/serialize.rs +++ b/src/io/json/write/serialize.rs @@ -6,6 +6,7 @@ use streaming_iterator::StreamingIterator; use crate::bitmap::utils::ZipValidity; use crate::datatypes::TimeUnit; use crate::io::iterator::BufStreamingIterator; +use crate::offset::Offset; use crate::temporal_conversions::{ date32_to_date, date64_to_date, timestamp_ms_to_datetime, timestamp_ns_to_datetime, timestamp_s_to_datetime, timestamp_us_to_datetime, diff --git a/src/io/json_integration/read/array.rs b/src/io/json_integration/read/array.rs index 90cd709cbf9..42fe220b96f 100644 --- a/src/io/json_integration/read/array.rs +++ b/src/io/json_integration/read/array.rs @@ -10,6 +10,7 @@ use crate::{ datatypes::{DataType, PhysicalType, PrimitiveType, Schema}, error::{Error, Result}, io::ipc::IpcField, + offset::Offset, types::{days_ms, i256, months_days_ns, NativeType}, }; diff --git a/src/io/odbc/write/serialize.rs b/src/io/odbc/write/serialize.rs index 01767e6d52f..7f2fc18aa7c 100644 --- a/src/io/odbc/write/serialize.rs +++ b/src/io/odbc/write/serialize.rs @@ -4,6 +4,7 @@ use crate::array::*; use crate::bitmap::Bitmap; use crate::datatypes::DataType; use crate::error::{Error, Result}; +use crate::offset::Offset; use crate::types::NativeType; use super::super::api; diff --git a/src/io/orc/read/mod.rs b/src/io/orc/read/mod.rs index ca6ce7354c3..4a365078236 100644 --- a/src/io/orc/read/mod.rs +++ b/src/io/orc/read/mod.rs @@ -1,12 +1,11 @@ //! APIs to read from [ORC format](https://orc.apache.org). use std::io::Read; -use crate::array::{ - Array, BinaryArray, BooleanArray, Int64Array, Offset, PrimitiveArray, Utf8Array, -}; +use crate::array::{Array, BinaryArray, BooleanArray, Int64Array, PrimitiveArray, Utf8Array}; use crate::bitmap::{Bitmap, MutableBitmap}; use crate::datatypes::{DataType, Field, Schema}; use crate::error::Error; +use crate::offset::Offset; use crate::types::NativeType; use orc_format::proto::stream::Kind; diff --git a/src/io/parquet/read/deserialize/binary/basic.rs b/src/io/parquet/read/deserialize/binary/basic.rs index 8790b43ea45..0a2aa098c45 100644 --- a/src/io/parquet/read/deserialize/binary/basic.rs +++ b/src/io/parquet/read/deserialize/binary/basic.rs @@ -9,11 +9,12 @@ use parquet2::{ }; use crate::{ - array::{Array, BinaryArray, Offset, Utf8Array}, + array::{Array, BinaryArray, Utf8Array}, bitmap::{Bitmap, MutableBitmap}, buffer::Buffer, datatypes::DataType, error::{Error, Result}, + offset::Offset, }; use super::super::utils::{ diff --git a/src/io/parquet/read/deserialize/binary/dictionary.rs b/src/io/parquet/read/deserialize/binary/dictionary.rs index 3000a7ca7d2..5cf3c07d97b 100644 --- a/src/io/parquet/read/deserialize/binary/dictionary.rs +++ b/src/io/parquet/read/deserialize/binary/dictionary.rs @@ -3,11 +3,12 @@ use std::collections::VecDeque; use parquet2::page::DictPage; use crate::{ - array::{Array, BinaryArray, DictionaryArray, DictionaryKey, Offset, Utf8Array}, + array::{Array, BinaryArray, DictionaryArray, DictionaryKey, Utf8Array}, bitmap::MutableBitmap, datatypes::{DataType, PhysicalType}, error::Result, io::parquet::read::deserialize::nested_utils::{InitNested, NestedState}, + offset::Offset, }; use super::super::Pages; diff --git a/src/io/parquet/read/deserialize/binary/nested.rs b/src/io/parquet/read/deserialize/binary/nested.rs index 8c0f5ef419e..2d345140db7 100644 --- a/src/io/parquet/read/deserialize/binary/nested.rs +++ b/src/io/parquet/read/deserialize/binary/nested.rs @@ -7,8 +7,8 @@ use parquet2::{ }; use crate::{ - array::Offset, bitmap::MutableBitmap, datatypes::DataType, error::Result, - io::parquet::read::Pages, + bitmap::MutableBitmap, datatypes::DataType, error::Result, io::parquet::read::Pages, + offset::Offset, }; use super::super::utils::MaybeNext; diff --git a/src/io/parquet/read/deserialize/binary/utils.rs b/src/io/parquet/read/deserialize/binary/utils.rs index a47cb967a55..d886c1bfae6 100644 --- a/src/io/parquet/read/deserialize/binary/utils.rs +++ b/src/io/parquet/read/deserialize/binary/utils.rs @@ -1,4 +1,4 @@ -use crate::array::Offset; +use crate::offset::Offset; use super::super::utils::Pushable; diff --git a/src/io/parquet/read/statistics/binary.rs b/src/io/parquet/read/statistics/binary.rs index 1786477e6bb..aeb43a6b3e0 100644 --- a/src/io/parquet/read/statistics/binary.rs +++ b/src/io/parquet/read/statistics/binary.rs @@ -1,7 +1,8 @@ -use crate::array::{MutableArray, MutableBinaryArray, Offset}; use parquet2::statistics::{BinaryStatistics, Statistics as ParquetStatistics}; +use crate::array::{MutableArray, MutableBinaryArray}; use crate::error::Result; +use crate::offset::Offset; pub(super) fn push( from: Option<&dyn ParquetStatistics>, diff --git a/src/io/parquet/read/statistics/utf8.rs b/src/io/parquet/read/statistics/utf8.rs index 7a447e2334f..da9fcb6e111 100644 --- a/src/io/parquet/read/statistics/utf8.rs +++ b/src/io/parquet/read/statistics/utf8.rs @@ -1,7 +1,8 @@ -use crate::array::{MutableArray, MutableUtf8Array, Offset}; use parquet2::statistics::{BinaryStatistics, Statistics as ParquetStatistics}; +use crate::array::{MutableArray, MutableUtf8Array}; use crate::error::Result; +use crate::offset::Offset; pub(super) fn push( from: Option<&dyn ParquetStatistics>, diff --git a/src/io/parquet/write/binary/basic.rs b/src/io/parquet/write/binary/basic.rs index 58156c901e6..f2a5071c14d 100644 --- a/src/io/parquet/write/binary/basic.rs +++ b/src/io/parquet/write/binary/basic.rs @@ -8,10 +8,11 @@ use parquet2::{ use super::super::utils; use super::super::WriteOptions; use crate::{ - array::{Array, BinaryArray, Offset}, + array::{Array, BinaryArray}, bitmap::Bitmap, error::{Error, Result}, io::parquet::read::schema::is_nullable, + offset::Offset, }; pub(crate) fn encode_plain( diff --git a/src/io/parquet/write/binary/nested.rs b/src/io/parquet/write/binary/nested.rs index 55f5d2ef247..950ea4190ca 100644 --- a/src/io/parquet/write/binary/nested.rs +++ b/src/io/parquet/write/binary/nested.rs @@ -6,8 +6,9 @@ use super::basic::{build_statistics, encode_plain}; use crate::io::parquet::read::schema::is_nullable; use crate::io::parquet::write::Nested; use crate::{ - array::{Array, BinaryArray, Offset}, + array::{Array, BinaryArray}, error::Result, + offset::Offset, }; pub fn array_to_page( diff --git a/src/io/parquet/write/nested/def.rs b/src/io/parquet/write/nested/def.rs index ea62c1c14dc..eb945a4c683 100644 --- a/src/io/parquet/write/nested/def.rs +++ b/src/io/parquet/write/nested/def.rs @@ -1,4 +1,4 @@ -use crate::{array::Offset, bitmap::Bitmap}; +use crate::{bitmap::Bitmap, offset::Offset}; use super::super::pages::{ListNested, Nested}; use super::rep::num_values; diff --git a/src/io/parquet/write/nested/mod.rs b/src/io/parquet/write/nested/mod.rs index d9f01e3f487..5f6cd4d3524 100644 --- a/src/io/parquet/write/nested/mod.rs +++ b/src/io/parquet/write/nested/mod.rs @@ -3,7 +3,7 @@ mod rep; use parquet2::{encoding::hybrid_rle::encode_u32, read::levels::get_bit_width, write::Version}; -use crate::{array::Offset, error::Result}; +use crate::{error::Result, offset::Offset}; use super::Nested; diff --git a/src/io/parquet/write/pages.rs b/src/io/parquet/write/pages.rs index caaa92866be..2259647d79c 100644 --- a/src/io/parquet/write/pages.rs +++ b/src/io/parquet/write/pages.rs @@ -1,10 +1,11 @@ use parquet2::schema::types::{ParquetType, PrimitiveType as ParquetPrimitiveType}; use parquet2::{page::Page, write::DynIter}; -use crate::array::{ListArray, Offset, StructArray}; +use crate::array::{ListArray, StructArray}; use crate::bitmap::Bitmap; use crate::datatypes::PhysicalType; use crate::io::parquet::read::schema::is_nullable; +use crate::offset::Offset; use crate::{ array::Array, error::{Error, Result}, diff --git a/src/io/parquet/write/utf8/basic.rs b/src/io/parquet/write/utf8/basic.rs index c1e7b9321bc..1f1aeaab8fd 100644 --- a/src/io/parquet/write/utf8/basic.rs +++ b/src/io/parquet/write/utf8/basic.rs @@ -9,9 +9,10 @@ use super::super::binary::{encode_delta, ord_binary}; use super::super::utils; use super::super::WriteOptions; use crate::{ - array::{Array, Offset, Utf8Array}, + array::{Array, Utf8Array}, error::{Error, Result}, io::parquet::read::schema::is_nullable, + offset::Offset, }; pub(crate) fn encode_plain( diff --git a/src/io/parquet/write/utf8/nested.rs b/src/io/parquet/write/utf8/nested.rs index 42babd46cd7..2792ef35712 100644 --- a/src/io/parquet/write/utf8/nested.rs +++ b/src/io/parquet/write/utf8/nested.rs @@ -6,8 +6,9 @@ use super::basic::{build_statistics, encode_plain}; use crate::io::parquet::read::schema::is_nullable; use crate::io::parquet::write::Nested; use crate::{ - array::{Array, Offset, Utf8Array}, + array::{Array, Utf8Array}, error::Result, + offset::Offset, }; pub fn array_to_page( diff --git a/src/lib.rs b/src/lib.rs index 64771b954cf..bef2e6e53c1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,6 +24,7 @@ pub mod error; #[cfg_attr(docsrs, doc(cfg(feature = "io_ipc")))] pub mod mmap; +pub mod offset; pub mod scalar; pub mod trusted_len; pub mod types; diff --git a/src/offset.rs b/src/offset.rs new file mode 100644 index 00000000000..edca7dc8b38 --- /dev/null +++ b/src/offset.rs @@ -0,0 +1,2 @@ +//! Contains the declaration of [`Offset`] +pub use crate::types::Offset; diff --git a/src/scalar/binary.rs b/src/scalar/binary.rs index 70a9be40374..cfbdbd04b0a 100644 --- a/src/scalar/binary.rs +++ b/src/scalar/binary.rs @@ -1,4 +1,4 @@ -use crate::{array::*, datatypes::DataType}; +use crate::{datatypes::DataType, offset::Offset}; use super::Scalar; diff --git a/src/scalar/list.rs b/src/scalar/list.rs index 438e13f7f55..98ee497d949 100644 --- a/src/scalar/list.rs +++ b/src/scalar/list.rs @@ -1,6 +1,6 @@ use std::any::Any; -use crate::{array::*, datatypes::DataType}; +use crate::{array::*, datatypes::DataType, offset::Offset}; use super::Scalar; diff --git a/src/scalar/utf8.rs b/src/scalar/utf8.rs index f62581eccbc..73ea98b729e 100644 --- a/src/scalar/utf8.rs +++ b/src/scalar/utf8.rs @@ -1,4 +1,4 @@ -use crate::{array::*, datatypes::DataType}; +use crate::{datatypes::DataType, offset::Offset}; use super::Scalar; diff --git a/src/temporal_conversions.rs b/src/temporal_conversions.rs index af643eaf7fd..8eae7213fc8 100644 --- a/src/temporal_conversions.rs +++ b/src/temporal_conversions.rs @@ -7,8 +7,9 @@ use chrono::{ use crate::error::Result; use crate::{ - array::{Offset, PrimitiveArray, Utf8Array}, + array::{PrimitiveArray, Utf8Array}, error::Error, + offset::Offset, }; use crate::{ datatypes::{DataType, TimeUnit}, diff --git a/src/util/bench_util.rs b/src/util/bench_util.rs index 3ba63ce37be..6ac87ed3f6a 100644 --- a/src/util/bench_util.rs +++ b/src/util/bench_util.rs @@ -3,7 +3,7 @@ use rand::distributions::{Alphanumeric, Distribution, Standard}; use rand::{rngs::StdRng, Rng, SeedableRng}; -use crate::{array::*, types::NativeType}; +use crate::{array::*, offset::Offset, types::NativeType}; /// Returns fixed seedable RNG pub fn seedable_rng() -> StdRng { diff --git a/tests/it/array/equal/utf8.rs b/tests/it/array/equal/utf8.rs index 2be9ebef83c..28e13d4a2ec 100644 --- a/tests/it/array/equal/utf8.rs +++ b/tests/it/array/equal/utf8.rs @@ -1,4 +1,5 @@ use arrow2::array::*; +use arrow2::offset::Offset; use super::{binary_cases, test_equal}; diff --git a/tests/it/compute/length.rs b/tests/it/compute/length.rs index 9bb37576956..0a6b5e51e8e 100644 --- a/tests/it/compute/length.rs +++ b/tests/it/compute/length.rs @@ -1,6 +1,7 @@ use arrow2::array::*; use arrow2::compute::length::*; use arrow2::datatypes::*; +use arrow2::offset::Offset; fn length_test_string() { vec![ diff --git a/tests/it/compute/regex_match.rs b/tests/it/compute/regex_match.rs index 141a87ad3e6..66f28d03b9b 100644 --- a/tests/it/compute/regex_match.rs +++ b/tests/it/compute/regex_match.rs @@ -1,6 +1,7 @@ -use arrow2::array::{BooleanArray, Offset, Utf8Array}; +use arrow2::array::{BooleanArray, Utf8Array}; use arrow2::compute::regex_match::*; use arrow2::error::Result; +use arrow2::offset::Offset; fn test_generic, &Utf8Array) -> Result>( lhs: Vec<&str>, diff --git a/tests/it/compute/substring.rs b/tests/it/compute/substring.rs index 365615cd51f..5b76a0ac348 100644 --- a/tests/it/compute/substring.rs +++ b/tests/it/compute/substring.rs @@ -1,4 +1,4 @@ -use arrow2::{array::*, compute::substring::*, error::Result}; +use arrow2::{array::*, compute::substring::*, error::Result, offset::Offset}; fn with_nulls_utf8() -> Result<()> { let cases = vec![ diff --git a/tests/it/compute/utf8.rs b/tests/it/compute/utf8.rs index 864dc0eca27..e9d8613ddeb 100644 --- a/tests/it/compute/utf8.rs +++ b/tests/it/compute/utf8.rs @@ -1,4 +1,4 @@ -use arrow2::{array::*, compute::utf8::*, error::Result}; +use arrow2::{array::*, compute::utf8::*, error::Result, offset::Offset}; fn with_nulls_utf8_lower() -> Result<()> { let cases = vec![ diff --git a/tests/it/io/parquet/mod.rs b/tests/it/io/parquet/mod.rs index 9097d32fb30..06afa74fc95 100644 --- a/tests/it/io/parquet/mod.rs +++ b/tests/it/io/parquet/mod.rs @@ -10,6 +10,7 @@ use arrow2::{ io::parquet::read as p_read, io::parquet::read::statistics::*, io::parquet::write::*, + offset::Offset, types::{days_ms, NativeType}, };