feat(rust, python): read decimal as f64 (#5938)

pola-rs · Dec 29, 2022 · 30da2b1 · 30da2b1
1 parent 73ef86c
commit 30da2b1
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 29 deletions.
diff --git a/polars/polars-core/src/chunked_array/cast.rs b/polars/polars-core/src/chunked_array/cast.rs
@@ -22,9 +22,10 @@ pub(crate) fn cast_chunks(
         }
     };
 
+    let arrow_dtype = dtype.to_arrow();
     let chunks = chunks
         .iter()
-        .map(|arr| arrow::compute::cast::cast(arr.as_ref(), &dtype.to_arrow(), options))
+        .map(|arr| arrow::compute::cast::cast(arr.as_ref(), &arrow_dtype, options))
         .collect::<arrow::error::Result<Vec<_>>>()?;
     Ok(chunks)
 }

diff --git a/polars/polars-core/src/series/from.rs b/polars/polars-core/src/series/from.rs
@@ -320,34 +320,6 @@ impl Series {
                 Ok(s)
             }
             #[cfg(feature = "dtype-struct")]
-            ArrowDataType::Map(_field, _sorted) => {
-                let arr = if chunks.len() > 1 {
-                    // don't spuriously call this. This triggers a read on mmaped data
-                    concatenate_owned_unchecked(&chunks).unwrap() as ArrayRef
-                } else {
-                    chunks[0].clone()
-                };
-                let arr = arr.as_any().downcast_ref::<MapArray>().unwrap();
-                // inner type is a struct
-                let struct_array = arr.field().clone();
-
-                // small list, because that's the maps offset dtype.
-                // means we are limited to i32::MAX rows
-                let data_type =
-                    ListArray::<i32>::default_datatype(struct_array.data_type().clone());
-                // physical representation of the map
-                let new_arr = ListArray::new(
-                    data_type.clone(),
-                    arr.offsets().clone(),
-                    struct_array,
-                    arr.validity().cloned(),
-                );
-                let mut chunks = chunks;
-                chunks.clear();
-                chunks.push(Box::new(new_arr));
-                Self::try_from_arrow_unchecked(name, chunks, &data_type)
-            }
-            #[cfg(feature = "dtype-struct")]
             ArrowDataType::Struct(_) => {
                 let arr = if chunks.len() > 1 {
                     // don't spuriously call this. This triggers a read on mmaped data
@@ -389,13 +361,35 @@ impl Series {
                     .collect::<PolarsResult<Vec<_>>>()?;
                 Ok(StructChunked::new_unchecked(name, &fields).into_series())
             }
+            ArrowDataType::Decimal(_, _) | ArrowDataType::Decimal256(_, _) => {
+                eprintln!(
+                    "Polars does not support decimal types so the 'Series' are read as Float64"
+                );
+                Ok(Float64Chunked::from_chunks(
+                    name,
+                    cast_chunks(&chunks, &DataType::Float64, true)?,
+                )
+                .into_series())
+            }
+            ArrowDataType::Map(_, _) => map_arrays_to_series(name, chunks),
             dt => Err(PolarsError::InvalidOperation(
                 format!("Cannot create polars series from {dt:?} type").into(),
             )),
         }
     }
 }
 
+fn map_arrays_to_series(name: &str, chunks: Vec<ArrayRef>) -> PolarsResult<Series> {
+    let chunks = chunks
+        .iter()
+        .map(|arr| {
+            let arr = arr.as_any().downcast_ref::<MapArray>().unwrap();
+            arr.field().clone()
+        })
+        .collect::<Vec<_>>();
+    Series::try_from((name, chunks))
+}
+
 fn convert_inner_types(arr: &ArrayRef) -> ArrayRef {
     match arr.data_type() {
         ArrowDataType::Utf8 => {