Initial support for serde/pickling expressions. (#3096)

* partial derive serde * python: first expression pickled
pola-rs · Apr 10, 2022 · 10301f6 · 10301f6
1 parent e1a6c59
commit 10301f6
Show file tree

Hide file tree

Showing 18 changed files with 473 additions and 374 deletions.
diff --git a/polars/Cargo.toml b/polars/Cargo.toml
@@ -28,6 +28,7 @@ default = [
 ndarray = ["polars-core/ndarray"]
 # serde support for dataframes and series
 serde = ["polars-core/serde"]
+serde-lazy = ["polars-core/serde-lazy", "polars-lazy/serde"]
 parquet = ["polars-io", "polars-core/parquet", "polars-lazy/parquet", "polars-io/parquet"]
 lazy = ["polars-core/lazy", "polars-lazy", "polars-lazy/compile"]
 # commented out until UB is fixed

diff --git a/polars/polars-arrow/Cargo.toml b/polars/polars-arrow/Cargo.toml
@@ -14,6 +14,7 @@ arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branc
 # arrow = { package = "arrow2", version = "0.10", default-features = false, features = ["compute_concatenate"] }
 hashbrown = "0.12"
 num = "^0.4"
+serde = { version = "1", features = ["derive"], optional = true }
 thiserror = "^1.0"
 
 [features]

diff --git a/polars/polars-arrow/src/kernels/rolling/no_nulls.rs b/polars/polars-arrow/src/kernels/rolling/no_nulls.rs
@@ -4,12 +4,15 @@ use arrow::array::{ArrayRef, PrimitiveArray};
 use arrow::datatypes::DataType;
 use arrow::types::NativeType;
 use num::{Float, NumCast, ToPrimitive, Zero};
+#[cfg(feature = "serde")]
+use serde::{Deserialize, Serialize};
 use std::any::Any;
 use std::fmt::Debug;
 use std::ops::{Add, Div, Mul, Sub};
 use std::sync::Arc;
 
 #[derive(Clone, Copy, PartialEq, Debug)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub enum QuantileInterpolOptions {
     Nearest,
     Lower,

diff --git a/polars/polars-core/Cargo.toml b/polars/polars-core/Cargo.toml
@@ -97,6 +97,8 @@ parquet = ["arrow/io_parquet"]
 # scale to terrabytes?
 bigidx = ["polars-arrow/bigidx"]
 
+serde-lazy = ["serde", "polars-arrow/serde"]
+
 docs-selection = [
   "ndarray",
   "is_in",

diff --git a/polars/polars-core/src/chunked_array/ops/mod.rs b/polars/polars-core/src/chunked_array/ops/mod.rs
@@ -47,6 +47,9 @@ pub(crate) mod unique;
 #[cfg(feature = "zip_with")]
 pub mod zip;
 
+#[cfg(feature = "serde-lazy")]
+use serde::{Deserialize, Serialize};
+
 #[cfg(feature = "to_list")]
 pub trait ToList<T: PolarsDataType> {
     fn to_list(&self) -> Result<ListChunked> {
@@ -503,6 +506,7 @@ pub trait ToDummies<T>: ChunkUnique<T> {
 }
 
 #[derive(Default, Copy, Clone, Eq, PartialEq, Debug)]
+#[cfg_attr(feature = "serde-lazy", derive(Serialize, Deserialize))]
 pub struct SortOptions {
     pub descending: bool,
     pub nulls_last: bool,

diff --git a/polars/polars-core/src/datatypes.rs b/polars/polars-core/src/datatypes.rs
@@ -611,6 +611,7 @@ impl PartialOrd for AnyValue<'_> {
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "serde-lazy", derive(Serialize, Deserialize))]
 pub enum TimeUnit {
     Nanoseconds,
     Microseconds,
@@ -658,6 +659,7 @@ impl TimeUnit {
 pub type TimeZone = String;
 
 #[derive(Clone, Debug)]
+#[cfg_attr(feature = "serde-lazy", derive(Serialize, Deserialize))]
 pub enum DataType {
     Boolean,
     UInt8,
@@ -686,11 +688,14 @@ pub enum DataType {
     #[cfg(feature = "object")]
     /// A generic type that can be used in a `Series`
     /// &'static str can be used to determine/set inner type
+    #[cfg_attr(feature = "serde-lazy", serde(skip))]
+    // how to deserialize a static?
     Object(&'static str),
     Null,
     #[cfg(feature = "dtype-categorical")]
     // The RevMapping has the internal state.
     // This is ignored with casts, comparisons, hashing etc.
+    #[cfg_attr(feature = "serde-lazy", serde(skip))]
     Categorical(Option<Arc<RevMapping>>),
     #[cfg(feature = "dtype-struct")]
     Struct(Vec<Field>),
@@ -823,6 +828,10 @@ impl PartialEq<ArrowDataType> for DataType {
 
 /// Characterizes the name and the [`DataType`] of a column.
 #[derive(Clone, Debug, PartialEq)]
+#[cfg_attr(feature = "serde-lazy", derive(Serialize, Deserialize))]
+// see: https://github.com/serde-rs/serde/issues/1712
+// we need this because `DataType` has a `&'static`
+#[cfg_attr(feature = "serde-lazy", serde(bound(deserialize = "'de: 'static")))]
 pub struct Field {
     name: String,
     dtype: DataType,

diff --git a/polars/polars-lazy/Cargo.toml b/polars/polars-lazy/Cargo.toml
@@ -96,6 +96,7 @@ glob = "0.3"
 parking_lot = "0.12"
 rayon = "1.5"
 regex = { version = "1.5", optional = true }
+serde = { version = "1", features = ["derive"], optional = true }
 
 polars-arrow = { version = "0.20.0", path = "../polars-arrow" }
 polars-core = { version = "0.20.0", path = "../polars-core", features = ["lazy", "private", "zip_with"], default-features = false }