Skip to content

Commit

Permalink
feat(pyspark): implement percent_rank and enable testing
Browse files Browse the repository at this point in the history
  • Loading branch information
cpcloud committed Mar 11, 2022
1 parent 77652a4 commit c051617
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 16 deletions.
6 changes: 1 addition & 5 deletions ibis/backends/pyspark/compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1313,11 +1313,7 @@ def compile_dense_rank(t, expr, scope, timecontext, **kwargs):

@compiles(ops.PercentRank)
def compile_percent_rank(t, expr, scope, timecontext, **kwargs):
raise com.UnsupportedOperationError(
'Pyspark percent_rank() function indexes from 0 '
'instead of 1, and does not match expected '
'output of ibis expressions.'
)
return F.percent_rank()


@compiles(ops.NTile)
Expand Down
17 changes: 6 additions & 11 deletions ibis/backends/tests/test_window.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,6 @@ def calc_zscore(s):
lambda t, win: t.id.percent_rank().over(win),
lambda t: t.id.rank(pct=True),
id='percent_rank',
marks=pytest.mark.notimpl(
["duckdb", "impala", "mysql", "postgres", "pyspark", "sqlite"]
),
),
param(
lambda t, win: t.float_col.ntile(buckets=7).over(win),
Expand All @@ -70,7 +67,7 @@ def calc_zscore(s):
id='last',
),
param(
lambda t, win: ibis.row_number().over(win),
lambda _, win: ibis.row_number().over(win),
lambda t: t.cumcount(),
id='row_number',
marks=pytest.mark.notimpl(["pandas"]),
Expand Down Expand Up @@ -180,7 +177,7 @@ def calc_zscore(s):
)
@pytest.mark.notimpl(["clickhouse", "dask", "datafusion"])
def test_grouped_bounded_expanding_window(
backend, alltypes, df, con, result_fn, expected_fn
backend, alltypes, df, result_fn, expected_fn
):

expr = alltypes.mutate(
Expand Down Expand Up @@ -228,7 +225,7 @@ def test_grouped_bounded_expanding_window(
# Some backends do not support non-grouped window specs
@pytest.mark.notimpl(["clickhouse", "dask", "datafusion"])
def test_ungrouped_bounded_expanding_window(
backend, alltypes, df, con, result_fn, expected_fn
backend, alltypes, df, result_fn, expected_fn
):

expr = alltypes.mutate(
Expand All @@ -248,7 +245,7 @@ def test_ungrouped_bounded_expanding_window(


@pytest.mark.notimpl(["clickhouse", "dask", "datafusion", "pandas"])
def test_grouped_bounded_following_window(backend, alltypes, df, con):
def test_grouped_bounded_following_window(backend, alltypes, df):

window = ibis.window(
preceding=0,
Expand Down Expand Up @@ -304,9 +301,7 @@ def test_grouped_bounded_following_window(backend, alltypes, df, con):
],
)
@pytest.mark.notimpl(["clickhouse", "dask", "datafusion"])
def test_grouped_bounded_preceding_window(
backend, alltypes, df, con, window_fn
):
def test_grouped_bounded_preceding_window(backend, alltypes, df, window_fn):
window = window_fn(alltypes)

expr = alltypes.mutate(val=alltypes.double_col.sum().over(window))
Expand Down Expand Up @@ -358,7 +353,7 @@ def test_grouped_bounded_preceding_window(
)
@pytest.mark.notimpl(["clickhouse", "datafusion"])
def test_grouped_unbounded_window(
backend, alltypes, df, con, result_fn, expected_fn, ordered
backend, alltypes, df, result_fn, expected_fn, ordered
):
# Define a window that is
# 1) Grouped
Expand Down

0 comments on commit c051617

Please sign in to comment.