recursive list builder in rows (#3293)

polars/polars-core/src/chunked_array/builder/list.rs

@@ -409,7 +409,13 @@ impl<'a> AnonymousListBuilder<'a> {
     }
 
     pub fn append_series(&mut self, s: &'a Series) {
-        self.builder.push_multiple(s.chunks());
+        match s.dtype() {
+            #[cfg(feature = "dtype-struct")]
+            DataType::Struct(_) => self.builder.push(&**s.array_ref(0)),
+            _ => {
+                self.builder.push_multiple(s.chunks());
+            }
+        }
     }
 
     pub fn finish(self) -> ListChunked {

polars/polars-core/src/frame/row.rs

@@ -1,4 +1,3 @@
-use crate::chunked_array::builder::get_list_builder;
 use crate::prelude::*;
 use crate::utils::get_supertype;
 use crate::POOL;
@@ -338,7 +337,6 @@ pub(crate) enum AnyValueBuffer<'a> {
     Float32(PrimitiveChunkedBuilder<Float32Type>),
     Float64(PrimitiveChunkedBuilder<Float64Type>),
     Utf8(Utf8ChunkedBuilder),
-    List(Box<dyn ListBuilderTrait>),
     All(Vec<AnyValue<'a>>),
 }
 
@@ -374,8 +372,7 @@ impl<'a> AnyValueBuffer<'a> {
             (Float64(builder), AnyValue::Null) => builder.append_null(),
             (Utf8(builder), AnyValue::Utf8(v)) => builder.append_value(v),
             (Utf8(builder), AnyValue::Null) => builder.append_null(),
-            (List(builder), AnyValue::List(v)) => builder.append_series(&v),
-            (List(builder), AnyValue::Null) => builder.append_null(),
+            // Struct and List can be recursive so use anyvalues for that
             (All(vals), v) => vals.push(v),
             _ => return None,
         };
@@ -405,7 +402,6 @@ impl<'a> AnyValueBuffer<'a> {
             Float32(b) => b.finish().into_series(),
             Float64(b) => b.finish().into_series(),
             Utf8(b) => b.finish().into_series(),
-            List(mut b) => b.finish().into_series(),
             All(vals) => Series::new("", vals),
         }
     }
@@ -433,7 +429,7 @@ impl From<(&DataType, usize)> for AnyValueBuffer<'_> {
             Float32 => AnyValueBuffer::Float32(PrimitiveChunkedBuilder::new("", len)),
             Float64 => AnyValueBuffer::Float64(PrimitiveChunkedBuilder::new("", len)),
             Utf8 => AnyValueBuffer::Utf8(Utf8ChunkedBuilder::new("", len, len * 5)),
-            List(inner) => AnyValueBuffer::List(get_list_builder(inner, len * 10, len, "")),
+            // Struct and List can be recursive so use anyvalues for that
             _ => AnyValueBuffer::All(Vec::with_capacity(len)),
         }
     }

py-polars/Cargo.toml

@@ -27,6 +27,7 @@ bincode = "1.3"
 libc = "0.2"
 ndarray = "0.15"
 numpy = "0.16"
+once_cell = "1"
 polars-core = { path = "../polars/polars-core", default-features = false }
 pyo3 = { version = "0.16", features = ["abi3-py37", "extension-module", "multiple-pymethods"] }
 serde_json = { version = "1", optional = true }

py-polars/src/conversion.rs

@@ -541,40 +541,32 @@ impl<'s> FromPyObject<'s> for Wrap<AnyValue<'s>> {
             }
             Ok(Wrap(AnyValue::StructOwned(Box::new((vals, keys)))))
         } else if ob.is_instance_of::<PyList>()? {
-            Python::with_gil(|py| {
-                let pypolars = PyModule::import(py, "polars").unwrap().to_object(py);
-                let series = pypolars.getattr(py, "Series").unwrap().call1(py, (ob,))?;
-                let py_pyseries = series.getattr(py, "_s").unwrap();
-                let series = py_pyseries.extract::<PySeries>(py).unwrap().series;
-                Ok(Wrap(AnyValue::List(series)))
-            })
+            let avs = ob.extract::<Wrap<Row>>()?.0;
+            let s = Series::new("", &avs.0);
+            Ok(Wrap(AnyValue::List(s)))
         } else if ob.hasattr("_s")? {
             let py_pyseries = ob.getattr("_s").unwrap();
             let series = py_pyseries.extract::<PySeries>().unwrap().series;
             Ok(Wrap(AnyValue::List(series)))
         } else if ob.get_type().name()?.contains("date") {
             let gil = Python::acquire_gil();
             let py = gil.python();
-            let pypolars = PyModule::import(py, "polars").unwrap().to_object(py);
-            let utils = pypolars.getattr(py, "utils").unwrap();
-            let utils = utils
+            let date = py_modules::UTILS
                 .getattr(py, "_date_to_pl_date")
                 .unwrap()
                 .call1(py, (ob,))
                 .unwrap();
-            let v = utils.extract::<i32>(py).unwrap();
+            let v = date.extract::<i32>(py).unwrap();
             Ok(Wrap(AnyValue::Date(v)))
         } else if ob.get_type().name()?.contains("timedelta") {
             let gil = Python::acquire_gil();
             let py = gil.python();
-            let pypolars = PyModule::import(py, "polars").unwrap().to_object(py);
-            let utils = pypolars.getattr(py, "utils").unwrap();
-            let utils = utils
+            let td = py_modules::UTILS
                 .getattr(py, "_timedelta_to_pl_timedelta")
                 .unwrap()
                 .call1(py, (ob, "us"))
                 .unwrap();
-            let v = utils.extract::<i64>(py).unwrap();
+            let v = td.extract::<i64>(py).unwrap();
             Ok(Wrap(AnyValue::Duration(v, TimeUnit::Microseconds)))
         } else {
             Err(PyErr::from(PyPolarsErr::Other(format!(

py-polars/src/lib.rs

@@ -29,6 +29,7 @@ pub mod lazy;
 mod list_construction;
 pub mod npy;
 pub mod prelude;
+pub(crate) mod py_modules;
 pub mod series;
 pub mod utils;
 

py-polars/src/prelude.rs

@@ -1,3 +1,4 @@
 pub use crate::conversion::*;
 pub use crate::datatypes::*;
+pub(crate) use crate::py_modules;
 pub use polars::prelude::*;

py-polars/src/py_modules.rs

@@ -0,0 +1,8 @@
+use once_cell::sync::Lazy;
+use pyo3::prelude::*;
+
+pub(crate) static POLARS: Lazy<PyObject> =
+    Lazy::new(|| Python::with_gil(|py| PyModule::import(py, "polars").unwrap().to_object(py)));
+
+pub(crate) static UTILS: Lazy<PyObject> =
+    Lazy::new(|| Python::with_gil(|py| POLARS.getattr(py, "utils").unwrap()));

py-polars/tests/test_struct.py

@@ -237,3 +237,13 @@ def test_from_dicts_struct() -> None:
     assert pl.from_dicts(
         [{"a": 1, "b": {"a_deep": 1, "b_deep": {"a_deeper": [1, 2, 4]}}}]
     ).to_series(1).to_list() == [{"a_deep": 1, "b_deep": {"a_deeper": [1, 2, 4]}}]
+
+    data = [
+        {"a": [{"b": 0, "c": 1}]},
+        {"a": [{"b": 1, "c": 2}]},
+    ]
+
+    assert pl.from_dicts(data).to_series().to_list() == [
+        [{"b": 0, "c": 1}],
+        [{"b": 1, "c": 2}],
+    ]