python; allow null columns in FFI

pola-rs · Oct 22, 2021 · 1b890e0 · 1b890e0
1 parent 7b9bf9e
commit 1b890e0
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 3 deletions.
diff --git a/py-polars/src/arrow_interop/to_rust.rs b/py-polars/src/arrow_interop/to_rust.rs
@@ -1,11 +1,13 @@
 use crate::error::PyPolarsEr;
+use crate::prelude::ArrowDataType;
 use polars_core::prelude::Arc;
 use polars_core::utils::arrow::{
     array::ArrayRef,
     datatypes::{Field, Schema},
     ffi,
     record_batch::RecordBatch,
 };
+use polars_core::utils::arrow::{array::PrimitiveArray, bitmap::MutableBitmap};
 use pyo3::ffi::Py_uintptr_t;
 use pyo3::prelude::*;
 
@@ -26,7 +28,17 @@ pub fn array_to_rust(obj: &PyAny) -> PyResult<ArrayRef> {
 
     unsafe {
         let field = ffi::import_field_from_c(schema.as_ref()).map_err(PyPolarsEr::from)?;
-        let array = ffi::import_array_from_c(array, &field).map_err(PyPolarsEr::from)?;
+        let array = if field.data_type == ArrowDataType::Null {
+            let mut validity = MutableBitmap::new();
+            validity.extend_constant(1, false);
+            Box::new(PrimitiveArray::from_data(
+                ArrowDataType::Float64,
+                vec![1.0f64].into(),
+                Some(validity.into()),
+            ))
+        } else {
+            ffi::import_array_from_c(array, &field).map_err(PyPolarsEr::from)?
+        };
         Ok(array.into())
     }
 }

diff --git a/py-polars/src/lazy/apply.rs b/py-polars/src/lazy/apply.rs
@@ -189,7 +189,7 @@ pub fn map_mul(
 
         // we return an error, because that will become a null value polars lazy apply list
         if apply_groups && out.is_none(py) {
-            return Err(PolarsError::NoData("".into()))
+            return Err(PolarsError::NoData("".into()));
         }
 
         // unpack the wrapper in a PySeries

diff --git a/py-polars/src/lib.rs b/py-polars/src/lib.rs
@@ -209,7 +209,7 @@ pub fn map_mul(
     output_type: &PyAny,
     apply_groups: bool,
 ) -> PyExpr {
-   lazy::map_mul(&pyexpr, py, lambda, output_type, apply_groups)
+    lazy::map_mul(&pyexpr, py, lambda, output_type, apply_groups)
 }
 
 #[pymodule]

diff --git a/py-polars/tests/test_interop.py b/py-polars/tests/test_interop.py
@@ -53,3 +53,10 @@ def test_arrow_list_chunked_array():
     ca = pa.chunked_array([a, a, a])
     s = pl.from_arrow(ca)
     assert s.dtype == pl.List
+
+
+def test_from_pandas_null():
+    test_df = pd.DataFrame([{0: None}, {0: None}])
+    out = pl.DataFrame(test_df)
+    assert out.dtypes == [pl.Float64]
+    assert out["0"][0] is None