Move migrations.

polars/polars-core/src/chunked_array/arithmetic.rs

@@ -64,7 +64,10 @@ where
         + Mul<Output = T::Native>
         + Div<Output = T::Native>
         + num::Zero,
-    Kernel: Fn(&PrimitiveArray<T>, &PrimitiveArray<T>) -> arrow::error::Result<PrimitiveArray<T>>,
+    Kernel: Fn(
+        &PrimitiveArray<T>,
+        &PrimitiveArray<T>,
+    ) -> arrow::error::Result<PrimitiveArray<T::Native>>,
     F: Fn(T::Native, T::Native) -> T::Native,
 {
     let mut ca = match (lhs.len(), rhs.len()) {

polars/polars-core/src/chunked_array/builder/mod.rs

@@ -4,10 +4,7 @@ use crate::{
     prelude::*,
     utils::{get_iter_capacity, NoNull},
 };
-use arrow::{
-    array::*,
-    buffer::{Buffer, MutableBuffer},
-};
+use arrow::{array::*, bitmap::Bitmap, buffer::MutableBuffer};
 use num::Num;
 use std::borrow::Cow;
 use std::iter::FromIterator;
@@ -72,7 +69,7 @@ where
     T: PolarsPrimitiveType,
     T::Native: Default,
 {
-    array_builder: Primitive<T>,
+    array_builder: Primitive<T::Native>,
     field: Field,
 }
 
@@ -199,7 +196,7 @@ impl ChunkedBuilder<Cow<'_, str>, Utf8Type> for Utf8ChunkedBuilderCow {
 }
 
 /// Get the null count and the null bitmap of the arrow array
-pub fn get_bitmap<T: Array + ?Sized>(arr: &T) -> (usize, Option<Buffer>) {
+pub fn get_bitmap<T: Array + ?Sized>(arr: &T) -> (usize, Option<Bitmap>) {
     let data = arr.data();
     (
         data.null_count(),
@@ -211,11 +208,11 @@ pub fn get_bitmap<T: Array + ?Sized>(arr: &T) -> (usize, Option<Buffer>) {
 }
 
 // Used in polars/src/chunked_array/apply.rs:24 to collect from aligned vecs and null bitmaps
-impl<T> FromIterator<(MutableBuffer<T::Native>, Option<Buffer>)> for ChunkedArray<T>
+impl<T> FromIterator<(MutableBuffer<T::Native>, Option<Bitmap>)> for ChunkedArray<T>
 where
     T: PolarsNumericType,
 {
-    fn from_iter<I: IntoIterator<Item = (MutableBuffer<T::Native>, Option<Buffer>)>>(
+    fn from_iter<I: IntoIterator<Item = (MutableBuffer<T::Native>, Option<Bitmap>)>>(
         iter: I,
     ) -> Self {
         let mut chunks = vec![];
@@ -358,7 +355,7 @@ pub struct ListPrimitiveChunkedBuilder<T>
 where
     T: PolarsPrimitiveType,
 {
-    pub builder: ListPrimitive<i32, Primitive<T>, T>,
+    pub builder: LargePrimitiveBuilder<T::Native>,
     field: Field,
 }
 
@@ -378,18 +375,19 @@ impl<T> ListPrimitiveChunkedBuilder<T>
 where
     T: PolarsPrimitiveType,
 {
-    pub fn new(name: &str, values_builder: Primitive<T>, capacity: usize) -> Self {
-        let builder = LargeListBuilder::with_capacity(values_builder, capacity);
+    pub fn new(name: &str, capacity: usize, values_capacity: usize) -> Self {
+        let builder =
+            LargePrimitiveBuilder::<T::Native>::with_capacities(capacity, values_capacity);
         let field = Field::new(name, DataType::List(T::get_dtype().to_arrow()));
 
-        ListPrimitiveChunkedBuilder { builder, field }
+        Self { builder, field }
     }
 
     pub fn append_slice(&mut self, opt_v: Option<&[T::Native]>) {
         match opt_v {
             Some(v) => {
                 self.builder.values().append_slice(v);
-                self.builder.append(true).expect("should not fail");
+                self.builder.push(true);
             }
             None => {
                 self.builder.append(false).expect("should not fail");
@@ -453,17 +451,18 @@ where
     }
 }
 
-type LargeListUtf8Builder = ListPrimitive<i64, Utf8Primitive<i64>, _>;
-type LargeListBooleanBuilder = ListPrimitive<i64, BooleanPrimitive, _>;
+type LargePrimitiveBuilder<T> = ListPrimitive<i64, Primitive<T>, T>;
+type LargeListUtf8Builder = ListPrimitive<i64, Utf8Primitive<i64>, &'static str>;
+type LargeListBooleanBuilder = ListPrimitive<i64, BooleanPrimitive, bool>;
 
 pub struct ListUtf8ChunkedBuilder {
     builder: LargeListUtf8Builder,
     field: Field,
 }
 
 impl ListUtf8ChunkedBuilder {
-    pub fn new(name: &str, capacity: usize) -> Self {
-        let builder = LargeListUtf8Builder::with_capacity(values_builder, capacity);
+    pub fn new(name: &str, capacity: usize, values_capacity: usize) -> Self {
+        let builder = LargeListUtf8Builder::with_capacities(capacity, values_capacity);
         let field = Field::new(name, DataType::List(ArrowDataType::LargeUtf8));
 
         ListUtf8ChunkedBuilder { builder, field }
@@ -506,13 +505,13 @@ impl ListBuilderTrait for ListUtf8ChunkedBuilder {
 }
 
 pub struct ListBooleanChunkedBuilder {
-    builder: LargeListBuilder<BooleanArrayBuilder>,
+    builder: LargeListBooleanBuilder,
     field: Field,
 }
 
 impl ListBooleanChunkedBuilder {
-    pub fn new(name: &str, values_builder: BooleanArrayBuilder, capacity: usize) -> Self {
-        let builder = LargeListBuilder::with_capacity(values_builder, capacity);
+    pub fn new(name: &str, capacity: usize, values_capacacity: usize) -> Self {
+        let builder = LargeListBooleanBuilder::with_capacities(capacity, values_capacacity);
         let field = Field::new(name, DataType::List(ArrowDataType::Boolean));
 
         Self { builder, field }
@@ -562,23 +561,19 @@ pub fn get_list_builder(
 ) -> Box<dyn ListBuilderTrait> {
     macro_rules! get_primitive_builder {
         ($type:ty) => {{
-            let values_builder = PrimitiveArrayBuilder::<$type>::new(value_capacity);
-            let builder = ListPrimitiveChunkedBuilder::new(&name, values_builder, list_capacity);
+            let builder = ListPrimitiveChunkedBuilder::new(&name, value_capacity);
             Box::new(builder)
         }};
     }
     macro_rules! get_bool_builder {
         () => {{
-            let values_builder = BooleanArrayBuilder::new(value_capacity);
-            let builder = ListBooleanChunkedBuilder::new(&name, values_builder, list_capacity);
+            let builder = ListBooleanChunkedBuilder::new(&name, list_capacity, value_capacity);
             Box::new(builder)
         }};
     }
     macro_rules! get_utf8_builder {
         () => {{
-            let values_builder =
-                LargeStringBuilder::with_capacity(value_capacity * 5, value_capacity);
-            let builder = ListUtf8ChunkedBuilder::new(&name, values_builder, list_capacity);
+            let builder = ListUtf8ChunkedBuilder::new(&name, list_capacity, 5 * value_capacity);
             Box::new(builder)
         }};
     }
@@ -608,8 +603,7 @@ mod test {
 
     #[test]
     fn test_list_builder() {
-        let values_builder = PrimitiveArrayBuilder::<Int32Type>::new(10);
-        let mut builder = ListPrimitiveChunkedBuilder::new("a", values_builder, 10);
+        let mut builder = ListPrimitiveChunkedBuilder::new("a", 10, 5);
 
         // create a series containing two chunks
         let mut s1 = Int32Chunked::new_from_slice("a", &[1, 2, 3]).into_series();
@@ -638,8 +632,7 @@ mod test {
 
     #[test]
     fn test_list_str_builder() {
-        let mut builder =
-            ListUtf8ChunkedBuilder::new("a", LargeStringBuilder::with_capacity(10, 10), 10);
+        let mut builder = ListUtf8ChunkedBuilder::new("a", 10, 10);
         builder.append_series(&Series::new("", &["foo", "bar"]));
         let ca = builder.finish();
         dbg!(ca);

polars/polars-core/src/chunked_array/cast.rs

@@ -291,12 +291,10 @@ impl ChunkCast for ListChunked {
 #[cfg(test)]
 mod test {
     use crate::prelude::*;
-    use polars_arrow::builder::PrimitiveArrayBuilder;
 
     #[test]
     fn test_cast_list() -> Result<()> {
-        let mut builder =
-            ListPrimitiveChunkedBuilder::<Int32Type>::new("a", PrimitiveArrayBuilder::new(10), 10);
+        let mut builder = ListPrimitiveChunkedBuilder::<Int32Type>::new("a", 10, 10);
         builder.append_slice(Some(&[1i32, 2, 3]));
         builder.append_slice(Some(&[1i32, 2, 3]));
         let ca = builder.finish();

polars/polars-core/src/chunked_array/comparison.rs

@@ -10,10 +10,13 @@ use std::sync::Arc;
 
 type LargeStringArray = Utf8Array<u64>;
 
-impl ChunkedArray {
+impl<T> ChunkedArray<T>
+where
+    T: PolarsNumericType,
+{
     /// First ensure that the chunks of lhs and rhs match and then iterates over the chunks and applies
     /// the comparison operator.
-    fn comparison<T: PolarsNumericType>(
+    fn comparison(
         &self,
         rhs: &ChunkedArray<T>,
         operator: impl Fn(&PrimitiveArray<T>, &PrimitiveArray<T>) -> arrow::error::Result<BooleanArray>,
@@ -446,13 +449,16 @@ impl NumComp for u16 {}
 impl NumComp for u32 {}
 impl NumComp for u64 {}
 
-impl<T, Rhs> ChunkedArray<T>
+impl<T> ChunkedArray<T>
 where
     T: PolarsNumericType,
     T::Native: NumCast,
-    Rhs: NumComp + ToPrimitive,
 {
-    fn primitive_compare_scalar(&self, rhs: Rhs, op: comparison::Operator) {
+    fn primitive_compare_scalar<Rhs: NumComp + ToPrimitive>(
+        &self,
+        rhs: Rhs,
+        op: comparison::Operator,
+    ) {
         let rhs = NumCast::from(rhs).expect("could not cast to underlying chunkedarray type");
         self.apply_kernel_cast(|arr| {
             Arc::new(comparison::primitive_compare_scalar(arr, rhs, op).unwrap())
@@ -495,7 +501,7 @@ where
     }
 }
 
-impl<T> Utf8Chunked {
+impl Utf8Chunked {
     fn utf8_compare_scalar(&self, rhs: &str, op: comparison::Operator) {
         self.apply_kernel_cast(|arr| {
             Arc::new(comparison::utf8_compare_scalar(arr, rhs, op).unwrap())

polars/polars-core/src/chunked_array/kernels/mod.rs

@@ -33,7 +33,7 @@ where
 /// Casts a `PrimitiveArray` to a different physical type and logical type.
 /// This operation is `O(N)`
 /// Values that do not fit in the new physical type are converted to nulls.
-pub(crate) fn cast_physical<S, T>(arr: &PrimitiveArray<S>, datatype: &DataType) -> ArrayRef
+pub(crate) fn cast_physical<S, T>(arr: &PrimitiveArray<S::Native>, datatype: &DataType) -> ArrayRef
 where
     S: PolarsNumericType,
     T: PolarsNumericType,
@@ -45,7 +45,7 @@ where
     Arc::new(array)
 }
 
-pub(crate) fn is_nan<T>(arr: &PrimitiveArray<T>) -> ArrayRef
+pub(crate) fn is_nan<T>(arr: &PrimitiveArray<T::Native>) -> ArrayRef
 where
     T: PolarsFloatType,
     T::Native: Float,
@@ -57,7 +57,7 @@ where
     Arc::new(BooleanArray::from_data(values, arr.validity().clone()))
 }
 
-pub(crate) fn is_not_nan<T>(arr: &PrimitiveArray<T>) -> ArrayRef
+pub(crate) fn is_not_nan<T>(arr: &PrimitiveArray<T::Native>) -> ArrayRef
 where
     T: PolarsFloatType,
     T::Native: Float,
@@ -69,7 +69,7 @@ where
     Arc::new(BooleanArray::from_data(values, arr.validity().clone()))
 }
 
-pub(crate) fn is_finite<T>(arr: &PrimitiveArray<T>) -> ArrayRef
+pub(crate) fn is_finite<T>(arr: &PrimitiveArray<T::Native>) -> ArrayRef
 where
     T: PolarsFloatType,
     T::Native: Float,
@@ -81,7 +81,7 @@ where
     Arc::new(BooleanArray::from_data(values, arr.validity().clone()))
 }
 
-pub(crate) fn is_infinite<T>(arr: &PrimitiveArray<T>) -> ArrayRef
+pub(crate) fn is_infinite<T>(arr: &PrimitiveArray<T::Native>) -> ArrayRef
 where
     T: PolarsFloatType,
     T::Native: Float,

polars/polars-core/src/chunked_array/kernels/take.rs

@@ -1,10 +1,11 @@
 use crate::prelude::*;
 use arrow::array::*;
 use arrow::buffer::MutableBuffer;
+use arrow::types::NativeType;
 use std::sync::Arc;
 
 /// Take kernel for single chunk without nulls and arrow array as index.
-pub(crate) unsafe fn take_no_null_primitive<T: PolarsNumericType>(
+pub(crate) unsafe fn take_no_null_primitive<T: NativeType>(
     arr: &PrimitiveArray<T>,
     indices: &UInt32Array,
 ) -> Arc<PrimitiveArray<T>> {
@@ -28,7 +29,7 @@ pub(crate) unsafe fn take_no_null_primitive<T: PolarsNumericType>(
 
 /// Take kernel for single chunk without nulls and an iterator as index.
 pub(crate) unsafe fn take_no_null_primitive_iter_unchecked<
-    T: PolarsNumericType,
+    T: NativeType,
     I: IntoIterator<Item = usize>,
 >(
     arr: &PrimitiveArray<T>,
@@ -48,7 +49,7 @@ pub(crate) unsafe fn take_no_null_primitive_iter_unchecked<
 }
 
 /// Take kernel for single chunk without nulls and an iterator as index that does bound checks.
-pub(crate) fn take_no_null_primitive_iter<T: PolarsNumericType, I: IntoIterator<Item = usize>>(
+pub(crate) fn take_no_null_primitive_iter<T: NativeType, I: IntoIterator<Item = usize>>(
     arr: &PrimitiveArray<T>,
     indices: I,
 ) -> Arc<PrimitiveArray<T>> {
@@ -66,10 +67,7 @@ pub(crate) fn take_no_null_primitive_iter<T: PolarsNumericType, I: IntoIterator<
 }
 
 /// Take kernel for a single chunk with null values and an iterator as index.
-pub(crate) unsafe fn take_primitive_iter_unchecked<
-    T: PolarsNumericType,
-    I: IntoIterator<Item = usize>,
->(
+pub(crate) unsafe fn take_primitive_iter_unchecked<T: NativeType, I: IntoIterator<Item = usize>>(
     arr: &PrimitiveArray<T>,
     indices: I,
 ) -> Arc<PrimitiveArray<T>> {
@@ -88,7 +86,7 @@ pub(crate) unsafe fn take_primitive_iter_unchecked<
 }
 
 /// Take kernel for a single chunk with null values and an iterator as index that does bound checks.
-pub(crate) fn take_primitive_iter<T: PolarsNumericType, I: IntoIterator<Item = usize>>(
+pub(crate) fn take_primitive_iter<T: NativeType, I: IntoIterator<Item = usize>>(
     arr: &PrimitiveArray<T>,
     indices: I,
 ) -> Arc<PrimitiveArray<T>> {
@@ -111,7 +109,7 @@ pub(crate) fn take_primitive_iter<T: PolarsNumericType, I: IntoIterator<Item = u
 /// Take kernel for a single chunk without nulls and an iterator that can produce None values.
 /// This is used in join operations.
 pub(crate) unsafe fn take_no_null_primitive_opt_iter_unchecked<
-    T: PolarsNumericType,
+    T: NativeType,
     I: IntoIterator<Item = Option<usize>>,
 >(
     arr: &PrimitiveArray<T>,
@@ -130,7 +128,7 @@ pub(crate) unsafe fn take_no_null_primitive_opt_iter_unchecked<
 /// Take kernel for a single chunk and an iterator that can produce None values.
 /// This is used in join operations.
 pub(crate) unsafe fn take_primitive_opt_iter_unchecked<
-    T: PolarsNumericType,
+    T: NativeType,
     I: IntoIterator<Item = Option<usize>>,
 >(
     arr: &PrimitiveArray<T>,

polars/polars-core/src/chunked_array/kernels/take_agg.rs

@@ -29,7 +29,7 @@ pub(crate) unsafe fn take_agg_primitive_iter_unchecked<
     I: IntoIterator<Item = usize>,
     F: Fn(T::Native, T::Native) -> T::Native,
 >(
-    arr: &PrimitiveArray<T>,
+    arr: &PrimitiveArray<T::Native>,
     indices: I,
     f: F,
     init: T::Native,
@@ -65,7 +65,7 @@ pub(crate) unsafe fn take_agg_primitive_iter_unchecked_count_nulls<
     I: IntoIterator<Item = usize>,
     F: Fn(T::Native, T::Native) -> T::Native,
 >(
-    arr: &PrimitiveArray<T>,
+    arr: &PrimitiveArray<T::Native>,
     indices: I,
     f: F,
     init: T::Native,