rust polars 0.20.0 (#2888)

polars/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "polars"
-version = "0.19.1"
+version = "0.20.0"
 authors = ["ritchie46 <ritchie46@gmail.com>"]
 edition = "2021"
 keywords = ["dataframe", "query-engine", "arrow"]
@@ -211,10 +211,10 @@ bench = [
 ]
 
 [dependencies]
-polars-core = { version = "0.19.1", path = "./polars-core", features = ["docs", "private"], default-features = false }
-polars-io = { version = "0.19.1", path = "./polars-io", features = ["private"], default-features = false, optional = true }
-polars-lazy = { version = "0.19.1", path = "./polars-lazy", features = ["private"], default-features = false, optional = true }
-polars-time = { version = "0.1.1", path = "./polars-time", default-features = false, optional = true }
+polars-core = { version = "0.20.0", path = "./polars-core", features = ["docs", "private"], default-features = false }
+polars-io = { version = "0.20.0", path = "./polars-io", features = ["private"], default-features = false, optional = true }
+polars-lazy = { version = "0.20.0", path = "./polars-lazy", features = ["private"], default-features = false, optional = true }
+polars-time = { version = "0.20.0", path = "./polars-time", default-features = false, optional = true }
 
 [dev-dependencies]
 ahash = "0.7"

polars/polars-arrow/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "polars-arrow"
-version = "0.19.1"
+version = "0.20.0"
 authors = ["ritchie46 <ritchie46@gmail.com>"]
 edition = "2021"
 license = "MIT"
@@ -10,8 +10,8 @@ description = "Arrow interfaces for Polars DataFrame library"
 
 [dependencies]
 # arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "81bfaddb92f432ae25bff4e9fdf200159ecebafe", default-features = false }
-arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "cherry_pick", default-features = false }
-# arrow = { package = "arrow2", version = "0.9", default-features = false, features = ["compute_concatenate"] }
+# arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "cherry_pick", default-features = false }
+arrow = { package = "arrow2", version = "0.10", default-features = false, features = ["compute_concatenate"] }
 hashbrown = "0.12"
 num = "^0.4"
 thiserror = "^1.0"

polars/polars-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "polars-core"
-version = "0.19.1"
+version = "0.20.0"
 authors = ["ritchie46 <ritchie46@gmail.com>"]
 edition = "2021"
 license = "MIT"
@@ -149,8 +149,8 @@ lazy_static = "1.4"
 ndarray = { version = "0.15", optional = true, default_features = false }
 num = "^0.4"
 num_cpus = "1"
-polars-arrow = { version = "0.19.1", path = "../polars-arrow", features = ["compute"] }
-polars-utils = { version = "0.1.0", path = "../polars-utils", optional = true }
+polars-arrow = { version = "0.20.0", path = "../polars-arrow", features = ["compute"] }
+polars-utils = { version = "0.20.0", path = "../polars-utils", optional = true }
 prettytable-rs = { version = "0.8.0", optional = true }
 rand = { version = "0.8", optional = true, features = ["small_rng", "std"] }
 rand_distr = { version = "0.4", optional = true }
@@ -164,10 +164,10 @@ thiserror = "^1.0"
 [dependencies.arrow]
 package = "arrow2"
 # git = "https://github.com/jorgecarleitao/arrow2"
-git = "https://github.com/ritchie46/arrow2"
+# git = "https://github.com/ritchie46/arrow2"
 # rev = "81bfaddb92f432ae25bff4e9fdf200159ecebafe"
-branch = "cherry_pick"
-# version = "0.9"
+# branch = "cherry_pick"
+version = "0.10"
 default-features = false
 features = [
   "compute_aggregate",

polars/polars-core/src/chunked_array/ops/sort/mod.rs

@@ -259,6 +259,7 @@ where
     }
 }
 
+#[cfg(feature = "sort_multiple")]
 fn argsort_multiple_numeric<T: PolarsNumericType>(
     ca: &ChunkedArray<T>,
     other: &[Series],

polars/polars-core/src/series/mod.rs

@@ -684,7 +684,7 @@ impl Series {
         }
     }
     /// Apply a rolling median to a Series. See:
-    /// [ChunkedArray::rolling_median](crate::prelude::ChunkWindow::rolling_median).
+    /// [`ChunkedArray::rolling_median`]
     #[cfg_attr(docsrs, doc(cfg(feature = "rolling_window")))]
     pub fn rolling_median(&self, _options: RollingOptions) -> Result<Series> {
         #[cfg(feature = "rolling_window")]
@@ -697,7 +697,7 @@ impl Series {
         }
     }
     /// Apply a rolling quantile to a Series. See:
-    /// [ChunkedArray::rolling_quantile](crate::prelude::ChunkWindow::rolling_quantile).
+    /// [`ChunkedArray::rolling_quantile`]
     #[cfg_attr(docsrs, doc(cfg(feature = "rolling_window")))]
     pub fn rolling_quantile(
         &self,
@@ -937,7 +937,7 @@ impl Series {
     }
 
     /// Compute the unique elements, but maintain order. This requires more work
-    /// than a naive [`Series::unique`].
+    /// than a naive [`Series::unique`](SeriesTrait::unique).
     pub fn unique_stable(&self) -> Result<Series> {
         let idx = self.arg_unique()?;
         // Safety:

polars/polars-io/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "polars-io"
-version = "0.19.1"
+version = "0.20.0"
 authors = ["ritchie46 <ritchie46@gmail.com>"]
 edition = "2021"
 license = "MIT"
@@ -35,8 +35,8 @@ private = []
 ahash = "0.7"
 anyhow = "1.0"
 # arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "81bfaddb92f432ae25bff4e9fdf200159ecebafe", default-features = false }
-arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "cherry_pick", default-features = false }
-# arrow = { package = "arrow2", version = "0.9", default-features = false }
+# arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "cherry_pick", default-features = false }
+arrow = { package = "arrow2", version = "0.10", default-features = false }
 csv-core = { version = "0.1.10", optional = true }
 dirs = "4.0"
 flate2 = { version = "1", optional = true, default-features = false }
@@ -46,9 +46,9 @@ memchr = "2.4"
 memmap = { package = "memmap2", version = "0.5.2", optional = true }
 num = "^0.4"
 num_cpus = "1"
-polars-arrow = { version = "0.19.1", path = "../polars-arrow" }
-polars-core = { version = "0.19.1", path = "../polars-core", features = ["private"], default-features = false }
-polars-utils = { version = "0.1.0", path = "../polars-utils", optional = true }
+polars-arrow = { version = "0.20.0", path = "../polars-arrow" }
+polars-core = { version = "0.20.0", path = "../polars-core", features = ["private"], default-features = false }
+polars-utils = { version = "0.20.0", path = "../polars-utils", optional = true }
 rayon = "1.5"
 regex = "1.5"
 serde_json = { version = "1", optional = true }

polars/polars-lazy/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "polars-lazy"
-version = "0.19.1"
+version = "0.20.0"
 authors = ["ritchie46 <ritchie46@gmail.com>"]
 edition = "2021"
 license = "MIT"
@@ -94,11 +94,11 @@ parking_lot = "0.12"
 rayon = "1.5"
 regex = { version = "1.5", optional = true }
 
-polars-arrow = { version = "0.19.1", path = "../polars-arrow" }
-polars-core = { version = "0.19.1", path = "../polars-core", features = ["lazy", "private", "zip_with"], default-features = false }
-polars-io = { version = "0.19.1", path = "../polars-io", features = ["lazy", "csv-file", "private"], default-features = false }
-polars-time = { version = "0.1.1", path = "../polars-time", optional = true }
-polars-utils = { version = "0.1.0", path = "../polars-utils" }
+polars-arrow = { version = "0.20.0", path = "../polars-arrow" }
+polars-core = { version = "0.20.0", path = "../polars-core", features = ["lazy", "private", "zip_with"], default-features = false }
+polars-io = { version = "0.20.0", path = "../polars-io", features = ["lazy", "csv-file", "private"], default-features = false }
+polars-time = { version = "0.20.0", path = "../polars-time", optional = true }
+polars-utils = { version = "0.20.0", path = "../polars-utils" }
 # uncomment to have datafusion integration
 # when uncommenting we both need to point to the same arrow version
 # datafusion = {version="4.0.0-SNAPSHOT", git = "https://github.com/apache/arrow-datafusion", rev = "88222b7", default-features=false, optional=true}

polars/polars-lazy/src/dsl/mod.rs

@@ -888,7 +888,7 @@ impl Expr {
 
     /// Apply a function/closure once the logical plan get executed.
     ///
-    /// This function is very similar to [`apply`], but differs in how it handles aggregations.
+    /// This function is very similar to [`Expr::apply`], but differs in how it handles aggregations.
     ///
     ///  * `map` should be used for operations that are independent of groups, e.g. `multiply * 2`, or `raise to the power`
     ///  * `apply` should be used for operations that work on a group of data. e.g. `sum`, `count`, etc.
@@ -916,7 +916,7 @@ impl Expr {
 
     /// Apply a function/closure once the logical plan get executed with many arguments
     ///
-    /// See the [`map`] function for the differences between [`map`] and [`apply`].
+    /// See the [`Expr::map`] function for the differences between [`map`](Expr::map) and [`apply`](Expr::apply).
     pub fn map_many<F>(self, function: F, arguments: &[Expr], output_type: GetOutput) -> Self
     where
         F: Fn(&mut [Series]) -> Result<Series> + 'static + Send + Sync,
@@ -1013,7 +1013,7 @@ impl Expr {
 
     /// Apply a function/closure over the groups with many arguments. This should only be used in a groupby aggregation.
     ///
-    /// See the [`apply`] function for the differences between [`map`] and [`apply`].
+    /// See the [`Expr::apply`] function for the differences between [`map`](Expr::map) and [`apply`](Expr::apply).
     pub fn apply_many<F>(self, function: F, arguments: &[Expr], output_type: GetOutput) -> Self
     where
         F: Fn(&mut [Series]) -> Result<Series> + 'static + Send + Sync,
@@ -1641,7 +1641,7 @@ impl Expr {
     }
 
     /// Apply a rolling median See:
-    /// [ChunkedArray::rolling_median](polars::prelude::ChunkWindow::rolling_median).
+    /// [`ChunkedArray::rolling_median`]
     #[cfg_attr(docsrs, doc(cfg(feature = "rolling_window")))]
     #[cfg(feature = "rolling_window")]
     pub fn rolling_median(self, options: RollingOptions) -> Expr {
@@ -1653,7 +1653,7 @@ impl Expr {
     }
 
     /// Apply a rolling quantile See:
-    /// [ChunkedArray::rolling_quantile](polars::prelude::ChunkWindow::rolling_quantile).
+    /// [`ChunkedArray::rolling_quantile`]
     #[cfg_attr(docsrs, doc(cfg(feature = "rolling_window")))]
     #[cfg(feature = "rolling_window")]
     pub fn rolling_quantile(

polars/polars-lazy/src/logical_plan/options.rs

@@ -103,10 +103,10 @@ pub struct FunctionOptions {
     /// We need to know this because we cannot see the difference between
     /// the following functions based on the output type and number of elements:
     ///
-    /// x: [1, 2, 3]
+    /// x: {1, 2, 3}
     ///
-    /// head_1(x) -> [1]
-    /// sum(x) -> [4]
+    /// head_1(x) -> {1}
+    /// sum(x) -> {4}
     pub(crate) auto_explode: bool,
     // used for formatting
     pub(crate) fmt_str: &'static str,

polars/polars-time/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "polars-time"
-version = "0.1.1"
+version = "0.20.0"
 authors = ["ritchie46 <ritchie46@gmail.com>"]
 edition = "2021"
 license = "MIT"
@@ -10,8 +10,8 @@ description = "Time related code for the polars dataframe library"
 
 [dependencies]
 chrono = "0.4"
-polars-arrow = { version = "0.19.0", path = "../polars-arrow", features = ["compute"] }
-polars-core = { version = "0.19.0", path = "../polars-core", features = ["temporal", "dtype-date", "dtype-datetime"] }
+polars-arrow = { version = "0.20.0", path = "../polars-arrow", features = ["compute"] }
+polars-core = { version = "0.20.0", path = "../polars-core", features = ["temporal", "dtype-date", "dtype-datetime"] }
 
 [features]
 dtype-date = []

polars/polars-time/src/date_range.rs

@@ -21,7 +21,7 @@ pub fn date_range_impl(
         .into_datetime(tu, None)
 }
 
-/// Create a [`DateTimeChunked`] from a given `start` and `stop` date and a given `every` interval.
+/// Create a [`DatetimeChunked`] from a given `start` and `stop` date and a given `every` interval.
 pub fn date_range(
     name: &str,
     start: NaiveDateTime,

polars/polars-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "polars-utils"
-version = "0.1.0"
+version = "0.20.0"
 authors = ["ritchie46 <ritchie46@gmail.com>"]
 edition = "2021"
 license = "MIT"

polars/src/lib.rs

@@ -45,6 +45,7 @@
 //!     - [ChunkedArray](#chunkedarray)
 //! * [SIMD](#simd)
 //! * [API](#api)
+//! * [Expressions](#expressions)
 //! * [Compile times](#compile-times)
 //! * [Performance](#performance-and-string-data)
 //!     - [Custom allocator](#custom-allocator)
@@ -64,9 +65,7 @@
 //! ### DataFrame
 //! A `DataFrame` is a 2 dimensional data structure that is backed by a `Series`, and it could be
 //! seen as an abstraction on `Vec<Series>`. Operations that can be executed on `DataFrame`s are very
-//! similar to what is done in a `SQL` like query. You can `GROUP`, `JOIN`, `PIVOT` etc. The
-//! closest arrow equivalent to a `DataFrame` is a [RecordBatch](https://docs.rs/arrow/4.0.0/arrow/record_batch/struct.RecordBatch.html),
-//! and Polars provides zero copy coercion.
+//! similar to what is done in a `SQL` like query. You can `GROUP`, `JOIN`, `PIVOT` etc.
 //!
 //! ### Series
 //! `Series` are the type agnostic columnar data representation of Polars. They provide many
@@ -91,6 +90,42 @@
 //! more verbose and less capable of building elegant composite queries. We recommend to use the Lazy API
 //! whenever you can.
 //!
+//! ## Expressions
+//! Polars has a powerful concept called expressions.
+//! Polars expressions can be used in various contexts and are a functional mapping of
+//! `Fn(Series) -> Series`, meaning that they have Series as input and Series as output.
+//! By looking at this functional definition, we can see that the output of an `Expr` also can serve
+//! as the input of an `Expr`.
+//!
+//! That may sound a bit strange, so lets give an example. The following is an expression:
+//!
+//! `col("foo").sort().head(2)`
+//!
+//! The snippet above says select column `"foo"` then sort this column and then take first 2 values
+//! of the sorted output.
+//! The power of expressions is that every expression produces a new expression and that they can
+//! be piped together.
+//! You can run an expression by passing them on one of polars execution contexts.
+//! Here we run two expressions in the **select** context:
+//!
+//! ```no_run
+//! # use polars::prelude::*;
+//! # let df = DataFrame::default();
+//!   df.lazy()
+//!    .select([
+//!        col("foo").sort(Default::default()).head(None),
+//!        col("bar").filter(col("foo").eq(lit(1))).sum(),
+//!    ])
+//!    .collect()?;
+//! ```
+//! All expressions are ran in parallel, meaning that separate polars expressions are embarrassingly parallel.
+//! (Note that within an expression there may be more parallelization going on).
+//!
+//! Understanding polars expressions is most important when starting with the polars library. Read more
+//! about them in the [User Guide](https://pola-rs.github.io/polars-book/user-guide/dsl/intro.html).
+//! Though the examples given there are in python. The expressions API is almost identical and the
+//! the read should certainly be valuable to rust users as well.
+//!
 //! ### Eager
 //! Read more in the pages of the following data structures /traits.
 //!
@@ -181,7 +216,7 @@
 //!     - `abs` - Get absolute values of Series
 //!     - `arange` - Range operation on Series
 //!     - `product` - Compute the product of a Series.
-//!     - `diff` - [`diff`] operation.
+//!     - `diff` - `diff` operation.
 //!     - `pct_change` - Compute change percentages.
 //! * `DataFrame` pretty printing (Choose one or none, but not both):
 //!     - `fmt` - Activate DataFrame formatting