Skip to content

Commit

Permalink
fix(rust, python): block ordered predicates before explode (#5951)
Browse files Browse the repository at this point in the history
  • Loading branch information
ritchie46 committed Dec 30, 2022
1 parent 053e70a commit 5c82197
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -373,9 +373,16 @@ impl PredicatePushDown {

Explode { input, columns, schema } => {
let condition = |name: Arc<str>| columns.iter().any(|s| s.as_str() == &*name);

// first columns that refer to the exploded columns should be done here
let mut local_predicates =
transfer_to_local_by_name(expr_arena, &mut acc_predicates, condition);
local_predicates.extend_from_slice(&transfer_to_local_by_node(&mut acc_predicates, |node| predicate_is_pushdown_boundary(node, expr_arena)));

// if any predicate is a pushdown boundary, thus influenced by order of predicates e.g.: sum(), over(), sort
// we do all here. #5950
if acc_predicates.values().chain(local_predicates.iter()).any(|node| predicate_is_pushdown_boundary(*node, expr_arena)) {
local_predicates.extend(acc_predicates.drain().map(|(_name, node)| node))
}

self.pushdown_and_assign(input, acc_predicates, lp_arena, expr_arena)?;
let lp = Explode { input, columns, schema };
Expand Down
16 changes: 16 additions & 0 deletions py-polars/tests/unit/test_filter.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,3 +77,19 @@ def test_is_in_bool() -> None:
assert df.filter(pl.col("A").is_in(bool_value_to_filter_on)).to_dict(False) == {
"A": [True, False]
}


def test_predicate_order_explode_5950() -> None:
df = pl.from_dict(
{
"i": [[0, 1], [1, 2]],
"n": [0, None],
}
)

assert (
df.lazy()
.explode("i")
.filter(pl.col("n").count().over(["i"]) == 2)
.filter(pl.col("n").is_not_null())
).collect().to_dict(False) == {"i": [1], "n": [0]}

0 comments on commit 5c82197

Please sign in to comment.