perf(rust): do not rechunk left joins (#5066)

pola-rs · Oct 1, 2022 · 649648e · 649648e
1 parent 66c98b9
commit 649648e
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 4 deletions.
diff --git a/polars/polars-core/src/frame/hash_join/mod.rs b/polars/polars-core/src/frame/hash_join/mod.rs
@@ -355,12 +355,18 @@ impl DataFrame {
 
         #[cfg(feature = "chunked_ids")]
         {
-            if _check_rechunk {
+            // a left join create chunked-ids
+            // the others not yet.
+            // TODO! change this to other join types once they support chunked-id joins
+            if _check_rechunk
+                && !(matches!(how, JoinType::Left)
+                    || std::env::var("POLARS_NO_CHUNKED_JOIN").is_ok())
+            {
                 let mut left = Cow::Borrowed(self);
                 let mut right = Cow::Borrowed(other);
                 if self.should_rechunk() {
                     if _verbose {
-                        eprintln!("join triggered a rechunk of the left dataframe: {} columns are affected", self.width());
+                        eprintln!("{:?} join triggered a rechunk of the left dataframe: {} columns are affected", how, self.width());
                     }
 
                     let mut tmp_left = self.clone();
@@ -369,7 +375,7 @@ impl DataFrame {
                 }
                 if other.should_rechunk() {
                     if _verbose {
-                        eprintln!("join triggered a rechunk of the right dataframe: {} columns are affected", other.width());
+                        eprintln!("{:?} join triggered a rechunk of the right dataframe: {} columns are affected", how, other.width());
                     }
                     let mut tmp_right = other.clone();
                     tmp_right.as_single_chunk_par();

diff --git a/polars/src/lib.rs b/polars/src/lib.rs
@@ -358,7 +358,9 @@
 //! * `POLARS_ALLOW_EXTENSION` -> allows for `[ObjectChunked<T>]` to be used in arrow, opening up possibilities like using
 //!                               `T` in complex lazy expressions. However this does require `unsafe` code allow this.
 //! * `POLARS_NO_PARQUET_STATISTICS` -> if set, statistics in parquet files are ignored.
-//! * `POLARS_PANIC_ON_ERR` -> panic instead of returning an Error..
+//! * `POLARS_PANIC_ON_ERR` -> panic instead of returning an Error.
+//! * `POLARS_NO_CHUNKED_JOIN` -> Force rechunk before joins.
+//!
 //!
 //! ## User Guide
 //! If you want to read more, [check the User Guide](https://pola-rs.github.io/polars-book/).