implement iterators for ObjectChunked

.github/workflows/test-windows.yaml

@@ -14,9 +14,6 @@ jobs:
           toolchain: nightly-2021-03-25
           override: true
       - name: Run tests
-        shell: bash
         run: |
-          # do not produce debug symbols to keep memory usage down
-          export RUSTFLAGS="-C debuginfo=0"
           cd polars && make test
           cd ../py-polars && ./tasks.sh build-run-tests

polars/polars-core/src/chunked_array/iterator/mod.rs

@@ -1,7 +1,5 @@
 use crate::datatypes::CategoricalChunked;
-use crate::prelude::{
-    BooleanChunked, ChunkedArray, ListChunked, PolarsNumericType, Series, Utf8Chunked,
-};
+use crate::prelude::*;
 use crate::utils::CustomIterTools;
 use arrow::array::{Array, BooleanArray, LargeListArray, LargeStringArray};
 use std::convert::TryFrom;
@@ -267,6 +265,31 @@ impl ListChunked {
     }
 }
 
+#[cfg(feature = "object")]
+impl<'a, T> IntoIterator for &'a ObjectChunked<T>
+where
+    T: PolarsObject,
+{
+    type Item = Option<&'a T>;
+    type IntoIter = Box<dyn PolarsIterator<Item = Self::Item> + 'a>;
+    fn into_iter(self) -> Self::IntoIter {
+        Box::new(self.downcast_iter().flatten().trust_my_length(self.len()))
+    }
+}
+
+#[cfg(feature = "object")]
+impl<T: PolarsObject> ObjectChunked<T> {
+    #[allow(clippy::wrong_self_convention)]
+    pub fn into_no_null_iter(
+        &self,
+    ) -> impl Iterator<Item = &T> + '_ + Send + Sync + ExactSizeIterator + DoubleEndedIterator {
+        self.downcast_iter()
+            .map(|arr| arr.values().iter())
+            .flatten()
+            .trust_my_length(self.len())
+    }
+}
+
 /// Trait for ChunkedArrays that don't have null values.
 /// The result is the most efficient implementation `Iterator`, according to the number of chunks.
 pub trait IntoNoNullIterator {

polars/polars-core/src/chunked_array/object/builder.rs

@@ -13,7 +13,7 @@ pub struct ObjectChunkedBuilder<T> {
 
 impl<T> ObjectChunkedBuilder<T>
 where
-    T: Any + Debug + Clone + Send + Sync + Default,
+    T: PolarsObject,
 {
     pub fn new(name: &str, capacity: usize) -> Self {
         ObjectChunkedBuilder {
@@ -24,17 +24,20 @@ where
     }
 
     /// Appends a value of type `T` into the builder
+    #[inline]
     pub fn append_value(&mut self, v: T) {
         self.values.push(v);
         self.bitmask_builder.append(true);
     }
 
     /// Appends a null slot into the builder
+    #[inline]
     pub fn append_null(&mut self) {
         self.values.push(T::default());
         self.bitmask_builder.append(false);
     }
 
+    #[inline]
     pub fn append_value_from_any(&mut self, v: &dyn Any) -> Result<()> {
         match v.downcast_ref::<T>() {
             None => Err(PolarsError::DataTypeMisMatch(
@@ -47,6 +50,7 @@ where
         }
     }
 
+    #[inline]
     pub fn append_option(&mut self, opt: Option<T>) {
         match opt {
             Some(s) => self.append_value(s),
@@ -84,7 +88,7 @@ where
 
 impl<T> Default for ObjectChunkedBuilder<T>
 where
-    T: Any + Debug + Clone + Send + Sync + Default,
+    T: PolarsObject,
 {
     fn default() -> Self {
         ObjectChunkedBuilder::new("", 0)
@@ -93,7 +97,7 @@ where
 
 impl<T> NewChunkedArray<ObjectType<T>, T> for ObjectChunked<T>
 where
-    T: Any + Debug + Clone + Send + Sync + Default,
+    T: PolarsObject,
 {
     fn new_from_slice(name: &str, v: &[T]) -> Self {
         Self::new_from_iter(name, v.iter().cloned())
@@ -124,7 +128,7 @@ where
 
 impl<T> ObjectChunked<T>
 where
-    T: Any + Debug + Clone + Send + Sync + Default,
+    T: PolarsObject,
 {
     pub fn new_from_vec(name: &str, v: Vec<T>) -> Self {
         let field = Arc::new(Field::new(name, DataType::Object));

polars/polars-core/src/chunked_array/object/iterator.rs

@@ -0,0 +1,84 @@
+use crate::chunked_array::object::{ObjectArray, PolarsObject};
+use arrow::array::Array;
+
+/// An iterator that returns Some(T) or None, that can be used on any ObjectArray
+// Note: This implementation is based on std's [Vec]s' [IntoIter].
+#[derive(Debug)]
+pub struct ObjectIter<'a, T: PolarsObject> {
+    array: &'a ObjectArray<T>,
+    current: usize,
+    current_end: usize,
+}
+
+impl<'a, T: PolarsObject> ObjectIter<'a, T> {
+    /// create a new iterator
+    pub fn new(array: &'a ObjectArray<T>) -> Self {
+        ObjectIter::<T> {
+            array,
+            current: 0,
+            current_end: array.len(),
+        }
+    }
+}
+
+impl<'a, T: PolarsObject> std::iter::Iterator for ObjectIter<'a, T> {
+    type Item = Option<&'a T>;
+
+    #[inline]
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.current == self.current_end {
+            None
+        } else if self.array.is_null(self.current) {
+            self.current += 1;
+            Some(None)
+        } else {
+            let old = self.current;
+            self.current += 1;
+            // Safety:
+            // we just checked bounds in `self.current_end == self.current`
+            // this is safe on the premise that this struct is initialized with
+            // current = array.len()
+            // and that current_end is ever only decremented
+            unsafe { Some(Some(self.array.value_unchecked(old))) }
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        (
+            self.array.len() - self.current,
+            Some(self.array.len() - self.current),
+        )
+    }
+}
+
+impl<'a, T: PolarsObject> std::iter::DoubleEndedIterator for ObjectIter<'a, T> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        if self.current_end == self.current {
+            None
+        } else {
+            self.current_end -= 1;
+            Some(if self.array.is_null(self.current_end) {
+                None
+            } else {
+                // Safety:
+                // we just checked bounds in `self.current_end == self.current`
+                // this is safe on the premise that this struct is initialized with
+                // current = array.len()
+                // and that current_end is ever only decremented
+                unsafe { Some(self.array.value_unchecked(self.current_end)) }
+            })
+        }
+    }
+}
+
+/// all arrays have known size.
+impl<'a, T: PolarsObject> std::iter::ExactSizeIterator for ObjectIter<'a, T> {}
+
+impl<'a, T: PolarsObject> IntoIterator for &'a ObjectArray<T> {
+    type Item = Option<&'a T>;
+    type IntoIter = ObjectIter<'a, T>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        ObjectIter::<'a, T>::new(self)
+    }
+}

polars/polars-core/src/chunked_array/object/mod.rs

@@ -1,4 +1,6 @@
 pub mod builder;
+mod iterator;
+
 pub use crate::prelude::*;
 use crate::utils::arrow::array::ArrayData;
 use arrow::array::{Array, ArrayRef, BooleanBufferBuilder, JsonEqual};
@@ -11,7 +13,7 @@ use std::sync::Arc;
 #[derive(Debug, Clone)]
 pub struct ObjectArray<T>
 where
-    T: Any + Debug + Clone + Send + Sync,
+    T: PolarsObject,
 {
     values: Arc<Vec<T>>,
     null_bitmap: Option<Arc<Bitmap>>,
@@ -20,18 +22,36 @@ where
     len: usize,
 }
 
+pub trait PolarsObject: Any + Debug + Clone + Send + Sync + Default {}
+
 impl<T> ObjectArray<T>
 where
-    T: Any + Debug + Clone + Send + Sync,
+    T: PolarsObject,
 {
+    /// Get a reference to the underlying data
+    pub fn values(&self) -> &Arc<Vec<T>> {
+        &self.values
+    }
+
+    /// Get a value at a certain index location
     pub fn value(&self, index: usize) -> &T {
         &self.values[self.offset + index]
     }
+
+    /// Get a value at a certain index location
+    ///
+    /// # Safety
+    ///
+    /// This does not any bound checks. The caller needs to ensure the index is within
+    /// the size of the array.
+    pub unsafe fn value_unchecked(&self, index: usize) -> &T {
+        &self.values.get_unchecked(index)
+    }
 }
 
 impl<T> JsonEqual for ObjectArray<T>
 where
-    T: Any + Debug + Clone + Send + Sync,
+    T: PolarsObject,
 {
     fn equals_json(&self, _json: &[&Value]) -> bool {
         false
@@ -40,7 +60,7 @@ where
 
 impl<T> Array for ObjectArray<T>
 where
-    T: Any + Debug + Clone + Send + Sync,
+    T: PolarsObject,
 {
     fn as_any(&self) -> &dyn Any {
         self
@@ -115,7 +135,7 @@ where
 
 impl<T> ObjectChunked<T>
 where
-    T: Any + Debug + Clone + Send + Sync + Default,
+    T: PolarsObject,
 {
     ///
     /// # Safety

polars/polars-core/src/chunked_array/ops/chunkops.rs

@@ -7,10 +7,6 @@ use arrow::array::Array;
 use arrow::compute::concat;
 use itertools::Itertools;
 use polars_arrow::prelude::*;
-#[cfg(feature = "object")]
-use std::any::Any;
-#[cfg(feature = "object")]
-use std::fmt::Debug;
 
 pub trait ChunkOps {
     /// Aggregate to contiguous memory.
@@ -93,7 +89,7 @@ impl ChunkOps for ListChunked {
 #[cfg(feature = "object")]
 impl<T> ChunkOps for ObjectChunked<T>
 where
-    T: Any + Debug + Clone + Send + Sync + Default,
+    T: PolarsObject,
 {
     fn rechunk(&self) -> Self
     where

polars/polars-core/src/chunked_array/ops/downcast.rs

@@ -89,9 +89,9 @@ impl ListChunked {
 }
 
 #[cfg(feature = "object")]
-impl<T> ObjectChunked<T>
+impl<'a, T> ObjectChunked<T>
 where
-    T: 'static + std::fmt::Debug + Clone + Send + Sync + Default,
+    T: PolarsObject,
 {
     pub fn downcast_iter(&self) -> impl Iterator<Item = &ObjectArray<T>> + DoubleEndedIterator {
         self.chunks.iter().map(|arr| {

polars/polars-core/src/chunked_array/ops/filter.rs

@@ -125,7 +125,7 @@ impl ChunkFilter<ListType> for ListChunked {
 #[cfg(feature = "object")]
 impl<T> ChunkFilter<ObjectType<T>> for ObjectChunked<T>
 where
-    T: 'static + std::fmt::Debug + Clone + Send + Sync + Default,
+    T: PolarsObject,
 {
     fn filter(&self, filter: &BooleanChunked) -> Result<ChunkedArray<ObjectType<T>>>
     where

polars/polars-core/src/prelude.rs

@@ -38,3 +38,6 @@ pub use std::sync::Arc;
 
 #[cfg(feature = "temporal")]
 pub use crate::chunked_array::temporal::conversion::*;
+
+#[cfg(feature = "object")]
+pub use crate::chunked_array::object::PolarsObject;

polars/polars-core/src/series/implementations/object.rs

@@ -6,12 +6,11 @@ use crate::series::private::PrivateSeries;
 use arrow::array::{ArrayData, ArrayRef};
 use arrow::buffer::Buffer;
 use std::any::Any;
-use std::fmt::Debug;
 
 #[cfg(feature = "object")]
 impl<T> IntoSeries for ObjectChunked<T>
 where
-    T: 'static + std::fmt::Debug + Clone + Send + Sync + Default,
+    T: PolarsObject,
 {
     fn into_series(self) -> Series {
         Series(Arc::new(SeriesWrap(self)))
@@ -20,15 +19,12 @@ where
 
 #[cfg(feature = "object")]
 #[cfg_attr(docsrs, doc(cfg(feature = "object")))]
-impl<T> PrivateSeries for SeriesWrap<ObjectChunked<T>> where
-    T: 'static + Debug + Clone + Send + Sync + Default
-{
-}
+impl<T> PrivateSeries for SeriesWrap<ObjectChunked<T>> where T: PolarsObject {}
 #[cfg(feature = "object")]
 #[cfg_attr(docsrs, doc(cfg(feature = "object")))]
 impl<T> SeriesTrait for SeriesWrap<ObjectChunked<T>>
 where
-    T: 'static + Debug + Clone + Send + Sync + Default,
+    T: PolarsObject,
 {
     fn rename(&mut self, name: &str) {
         ObjectChunked::rename(&mut self.0, name)

py-polars/src/dataframe.rs

@@ -74,7 +74,7 @@ impl PyDataFrame {
         mut n_threads: Option<usize>,
         path: Option<String>,
         overwrite_dtype: Option<Vec<(&str, &PyAny)>>,
-        low_memory: bool
+        low_memory: bool,
     ) -> PyResult<Self> {
         let encoding = match encoding {
             "utf8" => CsvEncoding::Utf8,

py-polars/src/lazy/dataframe.rs

@@ -85,7 +85,7 @@ impl PyLazyFrame {
         stop_after_n_rows: Option<usize>,
         cache: bool,
         overwrite_dtype: Option<Vec<(&str, &PyAny)>>,
-        low_memory: bool
+        low_memory: bool,
     ) -> Self {
         let delimiter = sep.as_bytes()[0];
 

py-polars/src/lazy/dsl.rs

@@ -149,7 +149,6 @@ impl PyExpr {
         self.clone().inner.arg_sort(reverse).into()
     }
 
-
     pub fn take(&self, idx: PyExpr) -> PyExpr {
         self.clone().inner.take(idx.inner).into()
     }