release memory on 0% selectivity (#4000)

polars/polars-arrow/Cargo.toml

@@ -11,7 +11,7 @@ description = "Arrow interfaces for Polars DataFrame library"
 [dependencies]
 # arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "98e49133b2e56e51e30335830485b3cf768eb5a2", features = ["compute_concatenate"], default-features = false }
 # arrow = { package = "arrow2", path = "../../../arrow2", features = ["compute_concatenate"], default-features = false }
-arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "polars", features = ["compute_concatenate"], default-features = false }
+arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "improve_filter", features = ["compute_concatenate"], default-features = false }
 # arrow = { package = "arrow2", version = "0.12", default-features = false, features = ["compute_concatenate"] }
 hashbrown = "0.12"
 num = "^0.4"

polars/polars-core/Cargo.toml

@@ -177,7 +177,7 @@ package = "arrow2"
 git = "https://github.com/ritchie46/arrow2"
 # rev = "98e49133b2e56e51e30335830485b3cf768eb5a2"
 # path = "../../../arrow2"
-branch = "polars"
+branch = "improve_filter"
 # version = "0.12"
 default-features = false
 features = [

polars/polars-core/src/chunked_array/bitwise.rs

@@ -132,10 +132,7 @@ impl BitOr for &BooleanChunked {
         let chunks = lhs
             .downcast_iter()
             .zip(rhs.downcast_iter())
-            .map(|(lhs, rhs)| {
-                Box::new(compute::boolean_kleene::or(lhs, rhs).expect("should be same size"))
-                    as ArrayRef
-            })
+            .map(|(lhs, rhs)| Box::new(compute::boolean_kleene::or(lhs, rhs)) as ArrayRef)
             .collect();
         BooleanChunked::from_chunks(self.name(), chunks)
     }
@@ -236,10 +233,7 @@ impl BitAnd for &BooleanChunked {
         let chunks = lhs
             .downcast_iter()
             .zip(rhs.downcast_iter())
-            .map(|(lhs, rhs)| {
-                Box::new(compute::boolean_kleene::and(lhs, rhs).expect("should be same size"))
-                    as ArrayRef
-            })
+            .map(|(lhs, rhs)| Box::new(compute::boolean_kleene::and(lhs, rhs)) as ArrayRef)
             .collect();
         BooleanChunked::from_chunks(self.name(), chunks)
     }

polars/polars-core/src/chunked_array/ops/filter.rs

@@ -76,7 +76,7 @@ impl ChunkFilter<Utf8Type> for Utf8Chunked {
         if filter.len() == 1 {
             return match filter.get(0) {
                 Some(true) => Ok(self.clone()),
-                _ => Ok(self.slice(0, 0)),
+                _ => Ok(Utf8Chunked::full_null(self.name(), 0)),
             };
         }
         check_filter_len!(self, filter);
@@ -98,7 +98,10 @@ impl ChunkFilter<ListType> for ListChunked {
         if filter.len() == 1 {
             return match filter.get(0) {
                 Some(true) => Ok(self.clone()),
-                _ => Ok(self.slice(0, 0)),
+                _ => Ok(ListChunked::from_chunks(
+                    self.name(),
+                    vec![new_empty_array(self.dtype().to_arrow())],
+                )),
             };
         }
         let (left, filter) = align_chunks_binary(self, filter);

polars/polars-io/Cargo.toml

@@ -38,7 +38,7 @@ private = ["polars-time/private"]
 ahash = "0.7"
 anyhow = "1.0"
 # arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "98e49133b2e56e51e30335830485b3cf768eb5a2", default-features = false }
-arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "polars", default-features = false }
+arrow = { package = "arrow2", git = "https://github.com/ritchie46/arrow2", branch = "improve_filter", default-features = false }
 # arrow = { package = "arrow2", version = "0.12", default-features = false }
 # arrow = { package = "arrow2", path = "../../../arrow2", default-features = false }
 csv-core = { version = "0.1.10", optional = true }